Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

33.3 KB
Raw Blame
from __future__ import annotations

import asyncio
import os
import sqlite3
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from threading import Barrier
from threading import Event as ThreadEvent
from unittest.mock import patch

import pytest
from sqlalchemy import create_engine
from sqlalchemy import event as sa_event
from sqlalchemy.engine import make_url
from sqlalchemy.schema import CreateSchema, DropSchema

from xagent.web.api.admin_users import delete_user
from xagent.web.models.database import (
    Base,
    get_db,
    get_engine,
    get_session_local,
    init_db,
)
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_command import TaskExecutionCommand
from xagent.web.models.task_command_terminal_event import TaskCommandTerminalEvent
from xagent.web.models.user import User
from xagent.web.services import task_command_transport as task_command_transport_module
from xagent.web.services.task_command_terminal_events import (
    FIRST_PARTY_MESSAGE_NOT_APPLIED_MESSAGE,
    FIRST_PARTY_MESSAGE_UNCONFIRMED_MESSAGE,
    TerminalTaskEventDraft,
    TerminalTaskEventMessageCode,
    first_party_message_terminal_text,
    stage_terminal_event,
)
from xagent.web.services.task_command_transport import (
    TaskCommandKind,
    claim_task_command,
    enqueue_task_command,
    fail_task_command,
    finish_task_command,
    retry_failed_task_command,
)


@pytest.fixture(
    params=(
        "sqlite",
        pytest.param("postgresql", marks=pytest.mark.postgresql),
    )
)
def db_session(request: pytest.FixtureRequest, tmp_path):
    admin_engine = None
    isolated_schema = None
    if request.param == "postgresql":
        raw_url = os.getenv("XAGENT_TEST_POSTGRES_URL") or os.getenv(
            "POSTGRES_TEST_DATABASE_URL"
        )
        if not raw_url:
            pytest.skip("XAGENT_TEST_POSTGRES_URL is not set")
        isolated_schema = f"xagent_terminal_events_{uuid.uuid4().hex}"
        admin_engine = create_engine(raw_url)
        with admin_engine.begin() as connection:
            connection.execute(CreateSchema(isolated_schema))
        db_url = str(
            make_url(raw_url).update_query_dict(
                {"options": f"-csearch_path={isolated_schema}"}
            )
        )
    else:
        db_url = f"sqlite:///{tmp_path / 'terminal-task-events.db'}"

    init_db(db_url=db_url)
    db = next(get_db())
    try:
        yield db
    finally:
        db.close()
        if admin_engine is None:
            Base.metadata.drop_all(bind=get_engine())
        else:
            get_engine().dispose()
            assert isolated_schema is not None
            with admin_engine.begin() as connection:
                connection.execute(DropSchema(isolated_schema, cascade=True))
            admin_engine.dispose()


def _create_running_task(db) -> tuple[User, Task]:
    user = User(username="terminal-event-user", password_hash="hash", is_admin=False)
    db.add(user)
    db.commit()
    task = Task(
        user_id=user.id,
        title="Terminal task events",
        description="Terminal task events",
        status=TaskStatus.RUNNING,
        execution_mode="auto",
        run_id="run-1",
        state_version=3,
        runner_id="worker-a",
        lease_expires_at=datetime.now(timezone.utc) + timedelta(minutes=1),
    )
    db.add(task)
    db.commit()
    db.refresh(task)
    return user, task


def _claim_command(
    db,
    user: User,
    task: Task,
    command_id: str,
    *,
    kind: TaskCommandKind = TaskCommandKind.PAUSE,
):
    enqueued = enqueue_task_command(
        db,
        task_id=int(task.id),
        actor_user_id=int(user.id),
        command_id=command_id,
        kind=kind,
        payload={"type": f"{kind.value}_task"},
    )
    claimed = claim_task_command(
        db,
        runner_id="worker-a",
        command_db_id=enqueued.command_id,
    )
    assert claimed is not None
    return claimed


def test_terminal_event_uses_the_command_acceptance_snapshot(db_session) -> None:
    accepted_owner, task = _create_running_task(db_session)
    accepted_owner_subject = accepted_owner.actor_subject
    assert accepted_owner_subject is not None
    enqueued = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=int(accepted_owner.id),
        command_id="stale-run-outcome",
        kind=TaskCommandKind.PAUSE,
        payload={"type": "pause_task"},
    )

    current_owner = User(
        username="terminal-event-reassigned-owner",
        password_hash="hash",
        is_admin=False,
    )
    db_session.add(current_owner)
    db_session.flush()
    task.user_id = int(current_owner.id)
    task.run_id = "run-2"
    task.state_version = 9
    db_session.commit()
    claimed = claim_task_command(
        db_session,
        runner_id="worker-a",
        command_db_id=enqueued.command_id,
    )
    assert claimed is not None
    assert fail_task_command(
        claimed.id,
        "worker-a",
        "internal detail",
        force_terminal=True,
        expected_attempt_count=claimed.attempt_count,
    )

    db_session.expire_all()
    command = db_session.get(TaskExecutionCommand, claimed.id)
    event = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
        .one()
    )
    assert command is not None
    assert command.target_run_id == "run-1"
    assert command.target_state_version == 3
    assert command.task_owner_user_id == int(accepted_owner.id)
    assert command.task_owner_subject == accepted_owner_subject
    assert event.task_run_id == "run-1"
    assert event.task_state_version == 3
    assert event.task_owner_user_id == int(accepted_owner.id)
    assert event.task_owner_subject == accepted_owner_subject


def test_terminal_event_refreshes_identity_mapped_command_after_bulk_update(
    db_session,
) -> None:
    user, task = _create_running_task(db_session)
    claimed = _claim_command(db_session, user, task, "identity-mapped-command")
    command = db_session.get(TaskExecutionCommand, claimed.id)
    assert command is not None
    assert command.status == "processing"

    updated = (
        db_session.query(TaskExecutionCommand)
        .filter(TaskExecutionCommand.id == claimed.id)
        .update(
            {TaskExecutionCommand.status: "completed"},
            synchronize_session=False,
        )
    )
    assert updated == 1
    assert command.status == "processing"

    event = stage_terminal_event(db_session, command_db_id=claimed.id)

    assert event.outcome == "completed"


@pytest.mark.parametrize("outcome", ["completed", "failed"])
def test_external_cancel_suppresses_command_identity_projection(
    db_session,
    outcome: str,
) -> None:
    user, task = _create_running_task(db_session)
    enqueued = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=int(user.id),
        command_id=f"external-{outcome}",
        kind=TaskCommandKind.CANCEL,
        payload={"type": "cancel_task", "scope": "external"},
    )
    claimed = claim_task_command(
        db_session,
        runner_id="worker-a",
        command_db_id=enqueued.command_id,
    )
    assert claimed is not None
    if outcome == "completed":
        assert finish_task_command(
            claimed.id,
            "worker-a",
            expected_attempt_count=claimed.attempt_count,
        )
    else:
        assert fail_task_command(
            claimed.id,
            "worker-a",
            "rejected",
            force_terminal=True,
            expected_attempt_count=claimed.attempt_count,
        )

    db_session.expire_all()
    event = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
        .one()
    )
    assert event.include_command_identity is False
    assert event.command_id == f"external-{outcome}"
    assert event.command_kind == TaskCommandKind.CANCEL.value
    assert event.actor_user_id == user.id
    assert event.task_owner_user_id == user.id
    assert event.task_run_id == "run-1"
    assert event.task_state_version == 3
    assert event.outcome_version == claimed.attempt_count


@pytest.mark.parametrize(
    ("kind", "payload", "with_actor", "expected_identity"),
    [
        pytest.param(
            TaskCommandKind.PAUSE,
            {"scope": "external"},
            True,
            True,
            id="external-scope-on-non-cancel",
        ),
        pytest.param(
            TaskCommandKind.CANCEL,
            {"type": "cancel_task"},
            True,
            True,
            id="cancel-without-scope",
        ),
        pytest.param(
            TaskCommandKind.CANCEL,
            {"scope": ["external"]},
            True,
            True,
            id="cancel-with-non-string-scope",
        ),
        pytest.param(
            TaskCommandKind.PAUSE,
            {"type": "pause_task"},
            False,
            True,
            id="actorless-command",
        ),
    ],
)
def test_terminal_event_identity_matches_kind_scope_and_actor(
    db_session,
    kind: TaskCommandKind,
    payload: dict,
    with_actor: bool,
    expected_identity: bool,
) -> None:
    user, task = _create_running_task(db_session)
    enqueued = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=int(user.id) if with_actor else None,
        command_id=f"identity-axis-{uuid.uuid4().hex}",
        kind=kind,
        payload=payload,
    )
    claimed = claim_task_command(
        db_session,
        runner_id="worker-a",
        command_db_id=enqueued.command_id,
    )
    assert claimed is not None
    assert finish_task_command(
        claimed.id,
        "worker-a",
        expected_attempt_count=claimed.attempt_count,
    )

    db_session.expire_all()
    event = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
        .one()
    )
    assert event.include_command_identity is expected_identity
    assert event.actor_user_id == (int(user.id) if with_actor else None)
    assert (event.actor_subject is not None) is with_actor


def test_terminal_event_preserves_explicit_identity_suppression(db_session) -> None:
    user, task = _create_running_task(db_session)
    claimed = _claim_command(db_session, user, task, "suppressed-command-identity")
    assert fail_task_command(
        claimed.id,
        "worker-a",
        "rejected",
        force_terminal=True,
        expected_attempt_count=claimed.attempt_count,
        terminal_event=TerminalTaskEventDraft(
            message_code=None,
            resend_safe=False,
            include_command_identity=False,
        ),
    )

    db_session.expire_all()
    event = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
        .one()
    )
    assert event.include_command_identity is False
    assert event.outcome == "failed"


def test_one_concurrent_claim_winner_stages_one_terminal_event(db_session) -> None:
    user, task = _create_running_task(db_session)
    task.runner_id = None
    task.lease_expires_at = None
    db_session.commit()
    enqueued = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=int(user.id),
        command_id="concurrent-terminal-event",
        kind=TaskCommandKind.RESUME,
        payload={"type": "resume_task"},
    )
    barrier = Barrier(2)

    def claim(runner_id: str):
        SessionLocal = get_session_local()
        with SessionLocal() as db:
            barrier.wait()
            return claim_task_command(
                db,
                runner_id=runner_id,
                command_db_id=enqueued.command_id,
            )

    with ThreadPoolExecutor(max_workers=2) as executor:
        attempts = [
            executor.submit(claim, "worker-a"),
            executor.submit(claim, "worker-b"),
        ]
        results = [attempt.result() for attempt in attempts]
    winners = [(index, result) for index, result in enumerate(results) if result]
    assert len(winners) == 1
    winner_index, winner = winners[0]
    assert winner is not None
    assert fail_task_command(
        winner.id,
        "worker-a" if winner_index == 0 else "worker-b",
        "internal detail",
        force_terminal=True,
        expected_attempt_count=winner.attempt_count,
    )

    db_session.expire_all()
    events = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == winner.id)
        .all()
    )
    assert len(events) == 1


def test_terminal_disposition_rolls_back_when_event_staging_fails(db_session) -> None:
    user, task = _create_running_task(db_session)
    claimed = _claim_command(
        db_session,
        user,
        task,
        "cancel-1",
        kind=TaskCommandKind.CANCEL,
    )

    with patch.object(
        task_command_transport_module,
        "stage_terminal_event",
        side_effect=RuntimeError("event write failed"),
    ):
        with pytest.raises(RuntimeError, match="event write failed"):
            fail_task_command(
                claimed.id,
                "worker-a",
                "internal detail",
                force_terminal=True,
                expected_attempt_count=claimed.attempt_count,
            )

    db_session.expire_all()
    command = db_session.get(TaskExecutionCommand, claimed.id)
    assert command is not None
    assert command.status == "processing"
    assert command.claimed_by == "worker-a"
    event_count = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
        .count()
    )
    assert event_count == 0

    db_session.rollback()
    assert fail_task_command(
        claimed.id,
        "worker-a",
        "retry after event write recovery",
        force_terminal=True,
        expected_attempt_count=claimed.attempt_count,
    )

    db_session.expire_all()
    recovered = db_session.get(TaskExecutionCommand, claimed.id)
    assert recovered is not None
    assert recovered.status == "failed"
    events = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
        .all()
    )
    assert len(events) == 1


def test_live_legacy_actor_replays_command_after_upgrade(db_session) -> None:
    if get_engine().dialect.name != "sqlite":
        pytest.skip("full parent-schema replay setup is SQLite-specific")

    actor, task = _create_running_task(db_session)
    actor_id = int(actor.id)
    task_id = int(task.id)
    first = enqueue_task_command(
        db_session,
        task_id=task_id,
        actor_user_id=actor_id,
        command_id="live-legacy-actor-replay",
        kind=TaskCommandKind.PAUSE,
        payload={"type": "pause_task"},
    )

    database_path = get_engine().url.database
    assert database_path is not None
    db_session.close()
    get_engine().dispose()
    legacy = sqlite3.connect(database_path)
    try:
        legacy.execute("DROP TABLE task_command_terminal_events")
        legacy.execute("ALTER TABLE task_execution_commands DROP COLUMN actor_subject")
        legacy.execute("DROP INDEX ix_users_actor_subject")
        legacy.execute("ALTER TABLE users DROP COLUMN actor_subject")
        legacy.execute(
            "ALTER TABLE task_execution_commands DROP COLUMN target_state_version"
        )
        legacy.execute(
            "UPDATE alembic_version SET version_num = ?",
            ("20260821_actor_oauth_flow_states",),
        )
        legacy.commit()
    finally:
        legacy.close()

    init_db(db_url=f"sqlite:///{database_path}")

    SessionLocal = get_session_local()
    with SessionLocal() as check:
        replay = enqueue_task_command(
            check,
            task_id=task_id,
            actor_user_id=actor_id,
            command_id="live-legacy-actor-replay",
            kind=TaskCommandKind.PAUSE,
            payload={"type": "pause_task"},
        )
        assert replay.command_id == first.command_id
        assert replay.created is False
        assert replay.payload_matches is True


@pytest.mark.parametrize(
    "replacement_before_upgrade",
    [False, True],
    ids=["orphan-remains", "actor-id-reused"],
)
def test_legacy_orphan_actor_does_not_block_terminal_disposition(
    db_session,
    replacement_before_upgrade: bool,
) -> None:
    if get_engine().dialect.name != "sqlite":
        pytest.skip("legacy orphan preservation is SQLite-specific")

    owner, task = _create_running_task(db_session)
    task_id = int(task.id)
    actor = User(
        username="terminal-event-admin-actor",
        password_hash="hash",
        is_admin=True,
    )
    db_session.add(actor)
    db_session.commit()
    actor_id = int(actor.id)
    first = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=actor_id,
        command_id="legacy-orphan-actor",
        kind=TaskCommandKind.PAUSE,
        payload={"type": "pause_task"},
    )
    second = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=int(owner.id),
        command_id="after-legacy-orphan-actor",
        kind=TaskCommandKind.RESUME,
        payload={"type": "resume_task"},
    )
    claimed = claim_task_command(
        db_session,
        runner_id="worker-a",
        command_db_id=first.command_id,
    )
    assert claimed is not None

    database_path = get_engine().url.database
    assert database_path is not None
    db_session.close()
    get_engine().dispose()
    legacy = sqlite3.connect(database_path)
    try:
        assert legacy.execute("PRAGMA foreign_keys").fetchone() == (0,)
        legacy.execute("DROP TABLE task_command_terminal_events")
        legacy.execute("ALTER TABLE task_execution_commands DROP COLUMN actor_subject")
        legacy.execute("DROP INDEX ix_users_actor_subject")
        legacy.execute("ALTER TABLE users DROP COLUMN actor_subject")
        legacy.execute(
            "ALTER TABLE task_execution_commands DROP COLUMN target_state_version"
        )
        legacy.execute("DELETE FROM users WHERE id = ?", (actor_id,))
        if replacement_before_upgrade:
            legacy.execute(
                "INSERT INTO users "
                "(id, username, password_hash, is_admin, created_at) "
                "VALUES (?, ?, ?, ?, ?)",
                (
                    actor_id,
                    "terminal-event-replacement",
                    "hash",
                    1,
                    "9999-12-31 23:59:59",
                ),
            )
        legacy.execute(
            "UPDATE alembic_version SET version_num = ?",
            ("20260821_actor_oauth_flow_states",),
        )
        legacy.commit()
    finally:
        legacy.close()

    init_db(db_url=f"sqlite:///{database_path}")
    assert fail_task_command(
        claimed.id,
        "worker-a",
        "executor failed",
        force_terminal=True,
        expected_attempt_count=claimed.attempt_count,
    )

    SessionLocal = get_session_local()
    with SessionLocal() as check:
        command = check.get(TaskExecutionCommand, claimed.id)
        assert command is not None
        assert command.status == "failed"
        assert command.failure_count == 1
        event = (
            check.query(TaskCommandTerminalEvent)
            .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
            .one()
        )
        assert event.actor_user_id == actor_id
        assert event.actor_subject == f"legacy-user-id:{actor_id}"
        replacement = check.get(User, actor_id)
        if replacement_before_upgrade:
            assert replacement is not None
            assert replacement.actor_subject is not None
            assert replacement.actor_subject != event.actor_subject
            replay = enqueue_task_command(
                check,
                task_id=task_id,
                actor_user_id=actor_id,
                command_id="legacy-orphan-actor",
                kind=TaskCommandKind.PAUSE,
                payload={"type": "pause_task"},
            )
            assert replay.created is False
            assert replay.payload_matches is False
            replacement_command = enqueue_task_command(
                check,
                task_id=task_id,
                actor_user_id=actor_id,
                command_id="replacement-actor-command",
                kind=TaskCommandKind.PAUSE,
                payload={"type": "pause_task"},
            )
            replacement_row = check.get(
                TaskExecutionCommand,
                replacement_command.command_id,
            )
            assert replacement_row is not None
            assert replacement_row.actor_subject == replacement.actor_subject
        else:
            assert replacement is None
        assert (
            claim_task_command(
                check,
                runner_id="worker-a",
                command_db_id=second.command_id,
            )
            is not None
        )


def test_legacy_orphan_task_owner_does_not_block_terminal_disposition(
    db_session,
) -> None:
    if get_engine().dialect.name != "sqlite":
        pytest.skip("legacy orphan preservation is SQLite-specific")

    owner, task = _create_running_task(db_session)
    actor = User(
        username="terminal-event-owner-orphan-actor",
        password_hash="hash",
        is_admin=True,
    )
    db_session.add(actor)
    db_session.commit()
    owner_id = int(owner.id)
    owner_subject = owner.actor_subject
    assert owner_subject is not None
    enqueued = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=int(actor.id),
        command_id="legacy-orphan-task-owner",
        kind=TaskCommandKind.PAUSE,
        payload={"type": "pause_task"},
    )
    claimed = claim_task_command(
        db_session,
        runner_id="worker-a",
        command_db_id=enqueued.command_id,
    )
    assert claimed is not None

    database_path = get_engine().url.database
    assert database_path is not None
    db_session.close()
    get_engine().dispose()
    legacy = sqlite3.connect(database_path)
    try:
        assert legacy.execute("PRAGMA foreign_keys").fetchone() == (0,)
        legacy.execute("DELETE FROM users WHERE id = ?", (owner_id,))
        legacy.commit()
    finally:
        legacy.close()

    assert fail_task_command(
        claimed.id,
        "worker-a",
        "executor failed",
        force_terminal=True,
        expected_attempt_count=claimed.attempt_count,
    )
    SessionLocal = get_session_local()
    with SessionLocal() as check:
        command = check.get(TaskExecutionCommand, claimed.id)
        event = (
            check.query(TaskCommandTerminalEvent)
            .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
            .one()
        )
        assert command is not None
        assert command.status == "failed"
        assert event.task_owner_user_id == owner_id
        assert event.task_owner_subject == owner_subject


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "delete_before_staging",
    [True, False],
    ids=["actor-deleted-before-staging", "actor-deleted-after-staging"],
)
async def test_actor_snapshot_survives_supported_user_deletion(
    db_session,
    delete_before_staging: bool,
) -> None:
    owner, task = _create_running_task(db_session)
    deleting_admin = User(
        username="terminal-event-deleting-admin",
        password_hash="hash",
        is_admin=True,
    )
    db_session.add(deleting_admin)
    db_session.flush()
    actor = User(
        username="terminal-event-cross-user-actor",
        password_hash="hash",
        is_admin=True,
    )
    db_session.add(actor)
    db_session.commit()
    actor_id = int(actor.id)
    enqueued = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=actor_id,
        command_id=f"actor-snapshot-{delete_before_staging}",
        kind=TaskCommandKind.PAUSE,
        payload={"type": "pause_task"},
    )
    accepted = db_session.get(TaskExecutionCommand, enqueued.command_id)
    assert accepted is not None
    actor_subject = str(accepted.actor_subject)
    assert actor_subject
    same_actor_command = enqueue_task_command(
        db_session,
        task_id=int(task.id),
        actor_user_id=actor_id,
        command_id=f"actor-snapshot-same-subject-{delete_before_staging}",
        kind=TaskCommandKind.RESUME,
        payload={"type": "resume_task"},
    )
    same_actor = db_session.get(TaskExecutionCommand, same_actor_command.command_id)
    assert same_actor is not None
    assert same_actor.actor_subject == actor_subject

    if delete_before_staging:
        assert await delete_user(actor_id, deleting_admin, db_session) == {
            "message": "User deleted successfully",
            "workspace_cleanup_pending": False,
            "external_cleanup_pending": False,
        }
    claimed = claim_task_command(
        db_session,
        runner_id="worker-a",
        command_db_id=enqueued.command_id,
    )
    assert claimed is not None
    assert finish_task_command(
        claimed.id,
        "worker-a",
        expected_attempt_count=claimed.attempt_count,
    )
    if not delete_before_staging:
        assert await delete_user(actor_id, deleting_admin, db_session) == {
            "message": "User deleted successfully",
            "workspace_cleanup_pending": False,
            "external_cleanup_pending": False,
        }
    if actor in db_session:
        db_session.expunge(actor)
    replacement = User(
        username="terminal-event-replacement-actor",
        password_hash="hash",
        is_admin=True,
    )
    db_session.add(replacement)
    db_session.commit()
    if get_engine().dialect.name == "sqlite":
        assert int(replacement.id) == actor_id
    assert replacement.actor_subject != actor_subject

    db_session.expire_all()
    command = db_session.get(TaskExecutionCommand, claimed.id)
    event = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
        .one()
    )
    assert command is not None
    assert command.actor_user_id is None
    assert command.actor_subject == actor_subject
    assert event.actor_user_id == (None if delete_before_staging else actor_id)
    assert event.actor_subject == actor_subject
    assert db_session.get(Task, task.id) is not None
    assert db_session.get(User, owner.id) is not None


def test_concurrent_first_commands_share_one_actor_subject(db_session) -> None:
    actor, first_task = _create_running_task(db_session)
    actor_id = int(actor.id)
    second_task = Task(
        user_id=actor.id,
        title="Concurrent terminal task events",
        description="Concurrent terminal task events",
        status=TaskStatus.RUNNING,
        execution_mode="auto",
        run_id="run-2",
        state_version=3,
        runner_id="worker-b",
        lease_expires_at=datetime.now(timezone.utc) + timedelta(minutes=1),
    )
    db_session.add(second_task)
    db_session.commit()
    db_session.query(User).filter(User.id == actor_id).update(
        {User.actor_subject: None},
        synchronize_session=False,
    )
    db_session.commit()

    barrier = Barrier(2)
    SessionLocal = get_session_local()

    def accept(task_id: int, command_id: str) -> str:
        with SessionLocal() as session:
            barrier.wait()
            enqueued = enqueue_task_command(
                session,
                task_id=task_id,
                actor_user_id=actor_id,
                command_id=command_id,
                kind=TaskCommandKind.PAUSE,
                payload={"type": "pause_task"},
            )
            command = session.get(TaskExecutionCommand, enqueued.command_id)
            assert command is not None
            assert command.actor_subject is not None
            assert command.task_owner_subject == command.actor_subject
            return str(command.actor_subject)

    with ThreadPoolExecutor(max_workers=2) as pool:
        subjects = list(
            pool.map(
                lambda item: accept(*item),
                [
                    (int(first_task.id), "concurrent-actor-subject-a"),
                    (int(second_task.id), "concurrent-actor-subject-b"),
                ],
            )
        )

    assert len(set(subjects)) == 1


@pytest.mark.asyncio
async def test_cancellation_after_commit_survives_a_fresh_database_session(
    db_session,
) -> None:
    user, task = _create_running_task(db_session)
    claimed = _claim_command(db_session, user, task, "cancel-after-commit")
    committed = ThreadEvent()
    release_worker = ThreadEvent()

    def persist_terminal_disposition() -> bool:
        persisted = fail_task_command(
            claimed.id,
            "worker-a",
            "internal detail",
            force_terminal=True,
            expected_attempt_count=claimed.attempt_count,
        )
        committed.set()
        assert release_worker.wait(timeout=5)
        return persisted

    persistence = asyncio.create_task(
        task_command_transport_module._persist_task_command_disposition(
            claimed,
            disposition="fail_task_command",
            operation=persist_terminal_disposition,
        )
    )
    assert await asyncio.to_thread(committed.wait, 2)
    persistence.cancel()
    release_worker.set()
    with pytest.raises(asyncio.CancelledError):
        await persistence

    db_url = str(get_engine().url)
    db_session.close()
    get_engine().dispose()
    init_db(db_url=db_url)
    fresh_db = next(get_db())
    try:
        events = (
            fresh_db.query(TaskCommandTerminalEvent)
            .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
            .all()
        )
        assert len(events) == 1
    finally:
        fresh_db.close()


def test_retry_appends_one_new_outcome_version_and_restaging_is_idempotent(
    db_session,
) -> None:
    user, task = _create_running_task(db_session)
    first_claim = _claim_command(db_session, user, task, "retry-terminal-outcome")
    assert fail_task_command(
        first_claim.id,
        "worker-a",
        "first terminal failure",
        force_terminal=True,
        expected_attempt_count=first_claim.attempt_count,
    )
    assert retry_failed_task_command(db_session, first_claim.id)
    second_claim = claim_task_command(
        db_session,
        runner_id="worker-a",
        command_db_id=first_claim.id,
    )
    assert second_claim is not None
    assert fail_task_command(
        second_claim.id,
        "worker-a",
        "second terminal failure",
        force_terminal=True,
        expected_attempt_count=second_claim.attempt_count,
    )

    marker_username = "caller-write-survives-terminal-event-conflict"
    db_session.add(User(username=marker_username, password_hash="hash", is_admin=False))
    db_session.flush()
    statements: list[str] = []

    def record_statement(_conn, _cursor, statement, *_args) -> None:
        statements.append(statement)

    engine = get_engine()
    sa_event.listen(engine, "before_cursor_execute", record_statement)
    try:
        restaged = stage_terminal_event(db_session, command_db_id=second_claim.id)
    finally:
        sa_event.remove(engine, "before_cursor_execute", record_statement)
    db_session.commit()
    db_session.expire_all()
    events = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == second_claim.id)
        .order_by(TaskCommandTerminalEvent.outcome_version)
        .all()
    )
    assert any("SAVEPOINT" in statement.upper() for statement in statements)
    assert any("ROLLBACK TO SAVEPOINT" in statement.upper() for statement in statements)
    assert [event.outcome_version for event in events] == [1, 2]
    assert restaged.id == events[1].id
    assert db_session.query(User).filter(User.username == marker_username).one()


def test_terminal_event_draft_uses_the_command_disposition_outcome(db_session) -> None:
    user, task = _create_running_task(db_session)
    claimed = _claim_command(
        db_session,
        user,
        task,
        "mismatched-outcome",
        kind=TaskCommandKind.CANCEL,
    )
    assert fail_task_command(
        claimed.id,
        "worker-a",
        "internal detail",
        force_terminal=True,
        expected_attempt_count=claimed.attempt_count,
        terminal_event=TerminalTaskEventDraft(
            message_code=None,
            resend_safe=False,
        ),
    )

    db_session.expire_all()
    event = (
        db_session.query(TaskCommandTerminalEvent)
        .filter(TaskCommandTerminalEvent.task_command_id == claimed.id)
        .one()
    )
    assert event.outcome == "failed"


def test_first_party_message_wording_asserts_non_application_only_on_proof() -> None:
    proven = TerminalTaskEventDraft(
        message_code=TerminalTaskEventMessageCode.TASK_COMMAND_DEFERRED,
        resend_safe=True,
    )
    assert (
        first_party_message_terminal_text(proven)
        == FIRST_PARTY_MESSAGE_NOT_APPLIED_MESSAGE
    )

    unproven = TerminalTaskEventDraft(
        message_code=TerminalTaskEventMessageCode.TASK_COMMAND_DEFERRED,
        resend_safe=False,
    )
    assert (
        first_party_message_terminal_text(unproven)
        == FIRST_PARTY_MESSAGE_UNCONFIRMED_MESSAGE
    )


def test_first_party_message_wording_treats_a_missing_draft_as_unknown() -> None:
    assert (
        first_party_message_terminal_text(None)
        == FIRST_PARTY_MESSAGE_UNCONFIRMED_MESSAGE
    )