Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

94.0 KB
Raw Blame
from __future__ import annotations

import asyncio
import sys
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch

import pytest

from core.message_output import HARNESS_PROMPT_ECHO_SPEC_KEY, terminal_turn_output
from core.native_dispatch_phase import (
    DISPATCH_PHASE_PREWRITE,
    backend_dispatch_attempted,
    mark_prewrite_user_stop,
    set_dispatch_phase,
)
from core.session_activities import SessionActivityRegistry
from modules.agents.catalog import AGENT_BACKENDS
from modules.agents import service as service_module
from modules.agents.service import AgentService
from modules.agents.codex.transport import CodexTransport
from modules.agents.base import AgentRequest
from modules.agents.opencode.poll_loop import OpenCodePollLoop
from modules.im import MessageContext
from storage.db import create_sqlite_engine
from storage.importer import ensure_sqlite_state
from storage.session_activities import SQLiteSessionActivityStore


class _RuntimeAgent:
    name = "claude"

    def __init__(self, release_first: asyncio.Event | None = None):
        self.started: list[str] = []
        self.release_first = release_first

    def runtime_turn_key(self, request):
        return request.composite_session_id

    async def handle_message(self, request):
        self.started.append(request.message)
        if request.message == "first" and self.release_first is not None:
            await self.release_first.wait()

    async def clear_sessions(self, _session_key):
        return 0

    async def handle_stop(self, _request):
        return False

    def record_runtime_turn_start(self, *, runtime_key, request):
        session_last_activity = getattr(self, "session_last_activity", None)
        if isinstance(session_last_activity, dict):
            previous = float(session_last_activity.get(runtime_key, 0.0) or 0.0)
            session_last_activity[runtime_key] = previous + 1.0


class _RaisingRuntimeAgent(_RuntimeAgent):
    async def handle_message(self, _request):
        raise RuntimeError("backend failed")


class _ClearingRuntimeAgent:
    name = "codex"

    def __init__(self, runtime_keys: set[str], on_clear=None):
        self.runtime_keys = runtime_keys
        self.clear_calls: list[str] = []
        self.on_clear = on_clear

    async def clear_sessions(self, session_key):
        self.clear_calls.append(session_key)
        if self.on_clear is not None:
            await self.on_clear()
        return 1

    def runtime_turn_keys_for_session_key(self, _session_key):
        return self.runtime_keys

    def runtime_turn_keys(self):
        return self.runtime_keys


class _StopRuntimeAgent(_RuntimeAgent):
    def __init__(self, reason: str):
        super().__init__()
        self.reason = reason

    async def handle_stop(self, request):
        request.stop_failure_reason = self.reason
        return False


class _RefreshingRuntimeAgent(_ClearingRuntimeAgent):
    def __init__(self, runtime_keys: set[str]):
        super().__init__(runtime_keys)
        self.refresh_calls: list[object] = []

    async def refresh_runtime_config(self, runtime_config):
        self.refresh_calls.append(runtime_config)


class _Controller:
    def __init__(self):
        self.session_turns = None


class _OrderRecordingTurnManager:
    def __init__(self, log: list[str]):
        self._log = log

    def on_running(self, _context):
        self._log.append("on_running")


class _OrderRecordingDispatcher:
    def __init__(self, log: list[str]):
        self._log = log

    async def begin_status_bubble(self, _context):
        self._log.append("begin_status_bubble")

    async def emit_harness_prompt(self, _context, text):
        self._log.append(f"echo:{text}")
        return "echo-msg"


class _OrderRecordingCheckpoints:
    def __init__(self, log: list[str]):
        self._log = log

    async def wait_for_turn_start(self, session_id):
        self._log.append(f"show_checkpoint:{session_id}")


class _TurnStartController:
    """Records the relative order of the turn-start hooks vs the agent run."""

    def __init__(self, log: list[str]):
        self.session_turns = _OrderRecordingTurnManager(log)
        self.message_dispatcher = _OrderRecordingDispatcher(log)
        self.show_git_checkpoint_service = _OrderRecordingCheckpoints(log)
        self._log = log

    def update_thread_message_id(self, _context):
        self._log.append("update_thread_message_id")

    @staticmethod
    def _session_id_from_context(_context):
        return "ses_order"


class _OrderRecordingAgent(_RuntimeAgent):
    def __init__(self, log: list[str]):
        super().__init__()
        self._log = log

    async def handle_message(self, request):
        self._log.append("handle_message")
        await super().handle_message(request)


def test_agent_service_runs_turn_start_hooks_after_gate_before_agent() -> None:
    async def _run():
        log: list[str] = []
        controller = _TurnStartController(log)
        service = AgentService(controller=controller)
        controller.agent_service = service
        agent = _OrderRecordingAgent(log)
        service.register(agent)

        request = _request("hi")
        set_dispatch_phase(request.context, DISPATCH_PHASE_PREWRITE)
        await service.handle_message("claude", request)

        # on_running (gate confirmed) must precede the bubble hooks, which in turn
        # precede the agent run. This is what keeps a queued turn from claiming
        # the trigger id / posting a premature bubble before it actually starts.
        # The Session's PRE_TURN Show checkpoint is waited for before the prompt
        # reaches the backend, so it snapshots the workspace before any edit.
        assert log == [
            "on_running",
            "update_thread_message_id",
            "begin_status_bubble",
            "show_checkpoint:ses_order",
            "handle_message",
        ]
        assert backend_dispatch_attempted(request.context) is False

    asyncio.run(_run())


def test_agent_service_defers_staged_harness_prompt_until_the_gate_is_held() -> None:
    """A queued Harness turn must not announce its prompt in the channel yet.

    The turn pipeline stages the prompt, but the turn may sit behind another turn on
    the runtime gate (or be cancelled there). Echoing at turn start keeps the channel
    reading trigger -> work -> result instead of showing the second task's prompt
    while the first task is still working (Codex P2).
    """

    async def _run():
        log: list[str] = []
        controller = _TurnStartController(log)
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        agent = _RuntimeAgent(release_first)
        service.register(agent)

        first_request = _request("first")
        first = asyncio.create_task(service.handle_message("claude", first_request))
        await asyncio.sleep(0)
        assert agent.started == ["first"]

        second_request = _request("second")
        second_request.context.platform_specific = {
            HARNESS_PROMPT_ECHO_SPEC_KEY: "summarize open PRs"
        }
        second = asyncio.create_task(service.handle_message("claude", second_request))
        await asyncio.sleep(0.05)
        assert "echo:summarize open PRs" not in log

        service.release_runtime_turn(first_request.context)
        release_first.set()
        await asyncio.wait_for(first, timeout=3)
        await asyncio.wait_for(second, timeout=3)

        # Echoed only once its OWN turn started (after the first turn's hooks), and
        # immediately ahead of that turn's status bubble.
        echo_at = log.index("echo:summarize open PRs")
        assert log[echo_at - 1] == "update_thread_message_id"
        assert log[echo_at + 1] == "begin_status_bubble"
        assert agent.started == ["first", "second"]
        # Popped, so a retry of the same request cannot post the prompt twice.
        assert HARNESS_PROMPT_ECHO_SPEC_KEY not in second_request.context.platform_specific

    asyncio.run(_run())


def test_agent_service_turn_start_without_a_staged_prompt_echoes_nothing() -> None:
    async def _run():
        log: list[str] = []
        controller = _TurnStartController(log)
        service = AgentService(controller=controller)
        controller.agent_service = service
        service.register(_OrderRecordingAgent(log))

        await service.handle_message("claude", _request("hi"))

        assert not any(entry.startswith("echo:") for entry in log)

    asyncio.run(_run())


def test_agent_service_failed_harness_prompt_echo_never_breaks_the_turn() -> None:
    async def _run():
        class _FailingEchoController:
            session_turns = None
            message_dispatcher = SimpleNamespace(
                emit_harness_prompt=AsyncMock(side_effect=RuntimeError("send failed")),
                begin_status_bubble=AsyncMock(),
            )

        controller = _FailingEchoController()
        service = AgentService(controller=controller)
        agent = _RuntimeAgent()
        service.register(agent)

        request = _request("hi")
        request.context.platform_specific = {HARNESS_PROMPT_ECHO_SPEC_KEY: "do the thing"}
        await service.handle_message("claude", request)

        assert agent.started == ["hi"]
        controller.message_dispatcher.begin_status_bubble.assert_awaited_once()

    asyncio.run(_run())


def test_agent_service_slow_harness_prompt_echo_does_not_hold_the_gate() -> None:
    """A degraded IM transport must not stall the turn it is announcing (Codex P2).

    The echo is awaited with the runtime gate held, and an adapter's own request
    budget is far longer than a turn start should ever wait (Telegram allows 60s), so
    the wait is bounded like the status-bubble post that follows it.
    """

    async def _run():
        started = asyncio.Event()

        async def _hanging_echo(_context, _text):
            started.set()
            await asyncio.sleep(30)

        class _SlowEchoController:
            session_turns = None
            message_dispatcher = SimpleNamespace(
                emit_harness_prompt=_hanging_echo,
                begin_status_bubble=AsyncMock(),
            )

        controller = _SlowEchoController()
        service = AgentService(controller=controller)
        agent = _RuntimeAgent()
        service.register(agent)

        request = _request("hi")
        request.context.platform_specific = {HARNESS_PROMPT_ECHO_SPEC_KEY: "do the thing"}
        with patch.object(service_module, "HARNESS_PROMPT_ECHO_TIMEOUT_SECONDS", 0.05):
            await asyncio.wait_for(service.handle_message("claude", request), timeout=3)

        assert started.is_set()
        assert agent.started == ["hi"]
        controller.message_dispatcher.begin_status_bubble.assert_awaited_once()

    asyncio.run(_run())


def test_agent_service_turn_start_hooks_optional_and_guarded() -> None:
    async def _run():
        # A controller WITHOUT the turn-start hooks (and a dispatcher whose
        # bubble raises) must not break the turn.
        class _NoHookController:
            session_turns = None
            message_dispatcher = SimpleNamespace(
                begin_status_bubble=AsyncMock(side_effect=RuntimeError("bubble boom"))
            )

        service = AgentService(controller=_NoHookController())
        agent = _RuntimeAgent()
        service.register(agent)

        await service.handle_message("claude", _request("hi"))

        assert agent.started == ["hi"]

    asyncio.run(_run())


class _FailingTurnManager:
    def on_running(self, _context):
        raise RuntimeError("status failed")


def _request(message: str, runtime_key: str = "session:/repo"):
    return SimpleNamespace(
        context=SimpleNamespace(platform_specific={}),
        message=message,
        composite_session_id=runtime_key,
        subagent_model=None,
        vibe_agent_model="fixture-avibe-model",
    )


@pytest.mark.parametrize("backend", sorted(AGENT_BACKENDS))
@pytest.mark.parametrize("model", [None, "", "   ", "default", "fixture-avibe-model", " padded-model "])
@pytest.mark.parametrize("selection_field", ["vibe_agent_model", "subagent_model"])
def test_model_selection_is_required_before_any_backend_dispatch(backend, model, selection_field):
    async def run():
        controller = _Controller()
        service = AgentService(controller=controller)
        controller.agent_service = service
        agent = _RuntimeAgent()
        agent.name = backend
        service.register(agent)
        request = _request("hello")
        setattr(request, selection_field, model)
        request.failure_handler = AsyncMock(return_value=True)
        selected = request.subagent_model if request.subagent_model is not None else request.vibe_agent_model
        if selected and selected.strip() and not (backend == "claude" and selected == "default"):
            await service.handle_message(backend, request)
            assert agent.started == ["hello"]
            assert (request.subagent_model or request.vibe_agent_model) == selected.strip()
            service.release_runtime_turn(request.context)
        else:
            with pytest.raises(ValueError, match="Select a model"):
                await service.handle_message(backend, request)
            assert agent.started == []
            assert not service.runtime_turn_active(request.composite_session_id)

    asyncio.run(run())


@pytest.mark.parametrize(
    ("backend", "hub_switch", "gateway_enabled", "has_source", "refusal", "fixed_in_models"),
    [
        ("vibey", "1", False, True, "The model gateway is off, so Vibey has no model to run on", True),
        ("vibey", "1", True, False, "Vibey has no model source yet", True),
        # A source but no model: the built-in backend's models all come from its Model Hub list.
        ("vibey", "1", True, True, "Select a model for the Vibey Agent", True),
        ("vibey", "0", True, True, "Model Hub is disabled on this instance, so Vibey has no model to run on", False),
        # A native backend runs direct without the Hub: choosing a model is still the fix.
        ("codex", "0", True, True, "Select a model for the Codex Agent", None),
    ],
)
def test_a_missing_model_the_model_hub_explains_names_the_model_hub(
    monkeypatch, backend, hub_switch, gateway_enabled, has_source, refusal, fixed_in_models
):
    """The Avibe Agent's models come only through the Model Hub: while it cannot supply any, choosing a model is not the fix."""
    from core.native_dispatch_phase import DISPATCH_PHASE_PREWRITE, prewrite_failure_evidence, set_dispatch_phase
    from modules.agents.model_hub import ModelSupplyRefusal
    from tests.scenario_harness.model_hub import config_with_sources, source

    monkeypatch.setenv("VIBE_MODEL_HUB_ENABLED", hub_switch)

    async def run():
        controller = _Controller()
        hub = config_with_sources([source("src_supply01", ["claude-opus-5-5"])] if has_source else [])
        hub.enabled = gateway_enabled
        controller.model_hub_service = SimpleNamespace(store=SimpleNamespace(load=lambda: hub))
        service = AgentService(controller=controller)
        controller.agent_service = service
        agent = _RuntimeAgent()
        agent.name = backend
        service.register(agent)
        request = _request("hello")
        request.vibe_agent_model = None
        request.failure_handler = AsyncMock(return_value=True)
        set_dispatch_phase(request.context, DISPATCH_PHASE_PREWRITE)

        with pytest.raises(ValueError, match=refusal) as refused:
            await service.handle_message(backend, request)
        assert agent.started == []
        # Whether Models is where the fix is decides the notice's next step.
        if fixed_in_models is None:
            assert not isinstance(refused.value, ModelSupplyRefusal)
            assert prewrite_failure_evidence(request.context) == {}
        else:
            assert isinstance(refused.value, ModelSupplyRefusal)
            assert refused.value.fixed_in_models is fixed_in_models
            # A supply refusal fails the same way until Models changes, so the input waits for an
            # explicit retry rather than spending the automatic startup retries.
            assert prewrite_failure_evidence(request.context) == {
                "reason": "model_supply_refusal", "requires_explicit_retry": True,
            }

    asyncio.run(run())


def test_agent_service_dispatches_runtime_config_refresh() -> None:
    service = AgentService(controller=SimpleNamespace())
    runtime_config = object()
    agent = SimpleNamespace(name="codex", refresh_runtime_config=AsyncMock())
    service.register(agent)

    handled = asyncio.run(service.refresh_runtime_config("codex", runtime_config))

    assert handled is True
    agent.refresh_runtime_config.assert_awaited_once_with(runtime_config)


def test_agent_service_reports_missing_runtime_refresh_contract() -> None:
    service = AgentService(controller=SimpleNamespace())
    service.register(SimpleNamespace(name="codex"))

    assert asyncio.run(service.refresh_runtime_config("codex", object())) is False
    assert asyncio.run(service.refresh_runtime_config("claude", object())) is False


def test_agent_service_notifies_run_owner_when_activity_runtime_disconnects() -> None:
    settle = Mock()
    controller = SimpleNamespace(
        scheduled_task_service=SimpleNamespace(settle_activity_runs=settle),
    )
    service = AgentService(controller)
    service.activities.start(
        backend="claude",
        runtime_key="runtime-1",
        session_id="session-1",
        activity_id="task-1",
        kind="background_task",
        run_id="run-1",
    )
    original_ack = service.activities.ack_recovered_terminal
    service.activities.ack_recovered_terminal = Mock(wraps=original_ack)

    service.end_activity_runtime("claude", "runtime-1")

    assert service.activities.active_for_runtime("claude", "runtime-1") == []
    activity = settle.call_args.args[0]
    assert activity.id == "task-1"
    assert activity.run_id == "run-1"
    assert activity.status == "disconnected"
    service.activities.ack_recovered_terminal.assert_called_once_with(activity)


def test_activity_runtime_disconnect_retains_snapshot_until_run_settles(
    tmp_path: Path,
) -> None:
    db_path = tmp_path / "state" / "vibe.sqlite"
    ensure_sqlite_state(db_path=db_path, primary_platform="avibe")
    engine = create_sqlite_engine(db_path)
    store = SQLiteSessionActivityStore(engine)
    controller = SimpleNamespace(
        scheduled_task_service=SimpleNamespace(
            settle_activity_runs=Mock(side_effect=RuntimeError("store unavailable")),
        ),
    )
    service = AgentService(
        controller,
        activities=SessionActivityRegistry(store),
    )
    service.activities.start(
        backend="claude",
        runtime_key="runtime-1",
        session_id="session-1",
        activity_id="task-1",
        kind="background_task",
        run_id="run-1",
    )

    service.end_activity_runtime("claude", "runtime-1")

    records = store.list_activities()
    assert len(records) == 1
    assert records[0]["phase"] == "terminal"
    assert records[0]["activity"]["status"] == "disconnected"
    recovered = SessionActivityRegistry(store)
    terminal = recovered.drain_recovered_terminals()
    assert [(item.id, item.run_id, item.status) for item in terminal] == [
        ("task-1", "run-1", "disconnected"),
    ]
    engine.dispose()


def test_activity_runtime_end_closes_unresolved_provenance_before_run_settlement():
    settled = []
    controller = SimpleNamespace(
        scheduled_task_service=SimpleNamespace(
            settle_activity_runs=settled.append,
        ),
    )
    service = AgentService(controller)
    service.activities.start(
        backend="claude",
        runtime_key="runtime-1",
        session_id="session-1",
        activity_id="task-failed",
        kind="background_task",
        metadata={"provenance_pending": True, "provenance_phase_id": "phase-1"},
    )
    service.activities.complete(
        backend="claude",
        runtime_key="runtime-1",
        activity_id="task-failed",
        status="failed",
        retain_terminal_snapshot=True,
    )

    completed = service.end_activity_runtime("claude", "runtime-1")

    assert [(activity.id, activity.status) for activity in completed] == [
        ("task-failed", "failed"),
    ]
    assert settled[0].metadata["provenance_generation_ended"] is True
    assert settled[0].metadata["provenance_unresolved"] is True
    assert service.activities.has_backend_work("claude") is False


def test_agent_service_serializes_same_runtime_until_terminal_release() -> None:
    async def _run():
        controller = _Controller()
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        agent = _RuntimeAgent(release_first)
        service.register(agent)

        first_request = _request("first")
        first = asyncio.create_task(service.handle_message("claude", first_request))
        await asyncio.sleep(0)
        assert agent.started == ["first"]

        second_request = _request("second")
        second = asyncio.create_task(service.handle_message("claude", second_request))
        await asyncio.sleep(0.05)
        assert agent.started == ["first"]

        service.release_runtime_turn(first_request.context)
        release_first.set()
        await asyncio.wait_for(first, timeout=3)
        await asyncio.wait_for(second, timeout=3)

        assert agent.started == ["first", "second"]

    asyncio.run(_run())


def test_hfr_142_gate_wait_does_not_refresh_backend_progress_clock() -> None:
    """HFR-142: a request waiting on the runtime gate creates no progress."""

    async def _run():
        first_started = asyncio.Event()
        second_waiting = asyncio.Event()

        class _ObservedRuntimeAgent(_RuntimeAgent):
            async def handle_message(self, request):
                self.started.append(request.message)
                if request.message == "first":
                    first_started.set()
                    if self.release_first is not None:
                        await self.release_first.wait()

        class _GateWaitIndicator:
            async def show_queued_reaction(self, _request):
                second_waiting.set()
                return True

            async def promote_reaction_to_running(
                self,
                _request,
                *,
                agent_name=None,
            ):
                return None

            async def finish(self, _request_or_handle):
                return None

        controller = _Controller()
        controller.processing_indicator = _GateWaitIndicator()
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        agent = _ObservedRuntimeAgent(release_first)
        agent.session_last_activity = {"session:/repo": 17.0}
        service.register(agent)

        first_request = _request("first")
        first = asyncio.create_task(service.handle_message("claude", first_request))
        await asyncio.wait_for(first_started.wait(), timeout=1)
        second_request = _request("second")
        second = asyncio.create_task(
            service.handle_message("claude", second_request)
        )
        await asyncio.wait_for(second_waiting.wait(), timeout=1)

        assert agent.started == ["first"]
        assert agent.session_last_activity == {"session:/repo": 17.0}

        service.mark_runtime_turn_started(first_request.context)
        assert agent.session_last_activity == {"session:/repo": 18.0}

        service.release_runtime_turn(first_request.context)
        release_first.set()
        await asyncio.wait_for(first, timeout=3)
        await asyncio.wait_for(second, timeout=3)
        assert agent.session_last_activity == {"session:/repo": 18.0}
        service.mark_runtime_turn_started(second_request.context)
        assert agent.session_last_activity == {"session:/repo": 19.0}

    asyncio.run(_run())


def test_agent_service_restart_barrier_preserves_same_runtime_fifo() -> None:
    async def _run():
        controller = _Controller()
        service = AgentService(controller=controller)
        controller.agent_service = service
        agent = _RuntimeAgent()
        service.register(agent)
        service.begin_backend_drain("claude")

        first_request = _request("first")
        second_request = _request("second")
        first = asyncio.create_task(service.handle_message("claude", first_request))
        await asyncio.sleep(0)
        second = asyncio.create_task(service.handle_message("claude", second_request))
        await asyncio.sleep(0.01)

        assert agent.started == []
        assert service.runtime_turn_tokens_for_backend("claude") == {}

        service.end_backend_drain("claude")
        await asyncio.sleep(0)
        assert agent.started == ["first"]

        service.release_runtime_turn(first_request.context)
        await asyncio.wait_for(first, timeout=3)
        await asyncio.wait_for(second, timeout=3)
        assert agent.started == ["first", "second"]

    asyncio.run(_run())


def test_agent_service_restart_wait_releases_gate_when_backend_disappears() -> None:
    async def _run():
        controller = _Controller()
        service = AgentService(controller=controller)
        controller.agent_service = service
        service.register(_RuntimeAgent())
        service.begin_backend_drain("claude")

        request = _request("waiting")
        task = asyncio.create_task(service.handle_message("claude", request))
        await asyncio.sleep(0.01)
        gate = service._turn_gates[request.composite_session_id]
        assert gate.lock.locked() is True
        assert gate.token == ""

        service.agents.pop("claude")
        service.end_backend_drain("claude")
        with pytest.raises(KeyError):
            await task
        assert gate.lock.locked() is False

    asyncio.run(_run())


def test_force_end_backend_activities_settles_pending_runs() -> None:
    """MH-MIG-012: the service ending an Activity itself names that cause, so
    its Runs settle as a runtime-refresh interruption rather than a silent kill."""
    settled = []
    controller = SimpleNamespace(
        scheduled_task_service=SimpleNamespace(
            settle_activity_runs=lambda activity, **kwargs: settled.append((activity, kwargs)),
        ),
    )
    service = AgentService(controller=controller)
    service.activities.start(
        backend="claude",
        runtime_key="runtime-1",
        session_id="ses-1",
        activity_id="task-active",
        kind="background_task",
        run_id="run-active",
    )
    service.activities.start(
        backend="claude",
        runtime_key="runtime-2",
        session_id="ses-2",
        activity_id="task-pending",
        kind="background_task",
        run_id="run-pending",
    )
    service.activities.complete(
        backend="claude",
        runtime_key="runtime-2",
        activity_id="task-pending",
        status="completed",
        expects_output=True,
    )
    original_ack = service.activities.ack_recovered_terminal
    service.activities.ack_recovered_terminal = Mock(wraps=original_ack)

    service.force_end_backend_activities("claude")

    assert sorted((item.id, item.status, item.metadata.get("interrupt_reason")) for item, _ in settled) == [
        ("task-active", "killed", "backend_refresh"),
        ("task-pending", "killed", "backend_refresh"),
    ]
    assert service.activities.ack_recovered_terminal.call_count == 2


def test_force_end_backend_activities_waits_for_run_settlement_before_ack() -> None:
    controller = SimpleNamespace(
        scheduled_task_service=SimpleNamespace(
            settle_activity_runs=Mock(side_effect=RuntimeError("store unavailable")),
        ),
    )
    service = AgentService(controller=controller)
    service.activities.start(
        backend="claude",
        runtime_key="runtime-1",
        session_id="ses-1",
        activity_id="task-active",
        kind="background_task",
        run_id="run-active",
    )
    original_ack = service.activities.ack_recovered_terminal
    service.activities.ack_recovered_terminal = Mock(wraps=original_ack)

    completed = service.force_end_backend_activities("claude")

    assert [(item.id, item.status) for item in completed] == [
        ("task-active", "killed"),
    ]
    service.activities.ack_recovered_terminal.assert_not_called()


def test_a_runtime_force_end_keeps_the_activity_until_its_run_settles() -> None:
    """A replaced runtime's killed Activity survives a failed Run settlement for a retry."""
    controller = SimpleNamespace(
        scheduled_task_service=SimpleNamespace(
            settle_activity_runs=Mock(side_effect=RuntimeError("store unavailable")),
        ),
    )
    service = AgentService(controller=controller)
    service.activities.start(
        backend="claude",
        runtime_key="runtime-1",
        session_id="ses-1",
        activity_id="task-active",
        kind="background_task",
        run_id="run-active",
    )

    completed = service.force_end_runtime_activities("claude", "runtime-1")

    assert [(item.id, item.status) for item in completed] == [("task-active", "killed")]
    assert [item.id for item in service.activities.terminal_snapshots_for_runtime("claude", "runtime-1")] == [
        "task-active"
    ]


def test_force_cancel_backend_turns_emits_terminal_before_release() -> None:
    async def _run():
        controller = _Controller()
        controller.emit_agent_message = AsyncMock()
        service = AgentService(controller=controller)
        controller.agent_service = service
        agent = _RuntimeAgent(asyncio.Event())
        service.register(agent)
        request = _request("first")
        task = asyncio.create_task(service.handle_message("claude", request))
        await asyncio.sleep(0)
        assert service.runtime_turn_tokens_for_backend("claude")

        await service.force_cancel_backend_turns("claude")

        assert task.cancelled()
        controller.emit_agent_message.assert_awaited_once_with(
            request.context,
            "result",
            "",
            is_error=True,
            level="silent",
            output=terminal_turn_output(),
        )
        assert service.runtime_turn_tokens_for_backend("claude") == {}

    asyncio.run(_run())


def test_a_runtime_force_end_spares_the_next_turn_admitted_while_it_settles() -> None:
    """Only the turns running when a runtime is force-ended are its work."""

    async def _run():
        controller = _Controller()
        controller.emit_agent_message = AsyncMock()
        service = AgentService(controller=controller)
        controller.agent_service = service
        agent = _RuntimeAgent()
        running: dict[str, asyncio.Event] = {}

        async def handle_message(request):
            running[request.message] = asyncio.Event()
            await running[request.message].wait()

        agent.handle_message = handle_message
        service.register(agent)
        first, second = _request("first"), _request("second")
        first.base_session_id = second.base_session_id = "s1"
        first_task = asyncio.create_task(service.handle_message("claude", first))
        await asyncio.sleep(0)
        second_task = asyncio.create_task(service.handle_message("claude", second))
        await asyncio.sleep(0)

        async def release_for_backend_refresh(**_kwargs):
            # The first turn finishes meanwhile, so the Session's next one starts.
            service.release_runtime_turn(first.context)
            running["first"].set()
            await first_task
            await asyncio.wait_for(_until_started("second"), 1)

        async def _until_started(message):
            while message not in running:
                await asyncio.sleep(0.01)

        controller.session_turns = SimpleNamespace(
            on_running=lambda _context: None,
            release_for_backend_refresh=release_for_backend_refresh,
        )

        await service.force_end_runtime_work("claude", base_session_ids={"s1"}, activation_identities=())

        assert not second_task.done()
        controller.emit_agent_message.assert_not_awaited()
        assert service.runtime_turn_tokens_for_backend("claude")
        running["second"].set()
        await asyncio.wait_for(second_task, 1)

    asyncio.run(_run())


def test_runtime_gen_006_a_disabled_agents_retried_stop_spares_the_re_enabled_agents_turn() -> None:
    """RUNTIME-GEN-006: a disabled agent's stop names a Session whose next turn runs on a new agent.

    The backend was enabled again before the retry, and the new agent now runs
    the Session's next turn: that turn is not the stopping agent's work.
    """

    async def _run():
        controller = _Controller()
        service = AgentService(controller=controller)
        controller.agent_service = service
        disabled, enabled = _RuntimeAgent(), _RuntimeAgent()
        running = asyncio.Event()

        async def handle_message(_request):
            running.set()
            await asyncio.Event().wait()

        enabled.handle_message = handle_message
        service.register(enabled)
        request = _request("next")
        request.base_session_id = "s1"
        task = asyncio.create_task(service.handle_message("claude", request))
        await asyncio.wait_for(running.wait(), 1)
        released = []

        async def release_for_backend_refresh(**kwargs):
            released.append(kwargs)

        controller.session_turns = SimpleNamespace(
            on_running=lambda _context: None,
            release_for_backend_refresh=release_for_backend_refresh,
        )

        await service.force_end_runtime_work(
            "claude",
            base_session_ids={"s1"},
            activation_identities=(),
            reason="backend_disabled",
            agent=disabled,
        )
        await asyncio.sleep(0)

        assert released == []
        assert not task.done()
        task.cancel()
        await asyncio.gather(task, return_exceptions=True)

    asyncio.run(_run())


class _RecordingIndicator:
    """Records the queued/promote/finish reaction calls the gate drives."""

    def __init__(self, log: list[str]):
        self._log = log

    async def show_queued_reaction(self, _request):
        self._log.append("show_queued")
        return True

    async def promote_reaction_to_running(self, _request, *, agent_name=None):
        self._log.append("promote")

    async def finish(self, _request_or_handle):
        self._log.append("finish")


class _IndicatorController(_Controller):
    def __init__(self, log: list[str]):
        super().__init__()
        self.processing_indicator = _RecordingIndicator(log)


def test_agent_service_shows_queued_reaction_then_promotes_on_start() -> None:
    async def _run():
        log: list[str] = []
        controller = _IndicatorController(log)
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        agent = _RuntimeAgent(release_first)
        service.register(agent)

        first_request = _request("first")
        first = asyncio.create_task(service.handle_message("claude", first_request))
        await asyncio.sleep(0)
        assert agent.started == ["first"]

        second = asyncio.create_task(service.handle_message("claude", _request("second")))
        await asyncio.sleep(0.05)
        # Only the SECOND message is queued (the first holds the gate): it shows the
        # queued 👌 and is blocked on acquire(). The first ran uncontended, so it
        # never showed 👌 and has already promoted to 👀.
        assert "show_queued" in log
        assert log.count("promote") == 1

        service.release_runtime_turn(first_request.context)
        release_first.set()
        await asyncio.wait_for(first, timeout=3)
        await asyncio.wait_for(second, timeout=3)

        assert log.count("show_queued") == 1  # only the genuinely-queued turn shows 👌
        assert log.count("promote") == 2  # both turns promoted when they started
        assert "finish" not in log  # no cancellation occurred

    asyncio.run(_run())


def test_agent_service_cleans_queued_reaction_when_cancelled_while_waiting() -> None:
    async def _run():
        log: list[str] = []
        controller = _IndicatorController(log)
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        agent = _RuntimeAgent(release_first)
        service.register(agent)

        first_request = _request("first")
        first = asyncio.create_task(service.handle_message("claude", first_request))
        await asyncio.sleep(0)
        assert agent.started == ["first"]

        second = asyncio.create_task(service.handle_message("claude", _request("second")))
        await asyncio.sleep(0.05)
        assert "show_queued" in log

        # Cancel the queued message while it is still blocked on gate.acquire():
        # its queued 👌 must be cleaned up (P0.1), and it must never promote.
        second.cancel()
        try:
            await second
        except asyncio.CancelledError:
            pass

        assert "finish" in log
        assert log.count("promote") == 1

        first.cancel()
        await asyncio.gather(first, return_exceptions=True)

    asyncio.run(_run())


class _ReactionIM:
    """Records the real reaction add/remove calls per message id."""

    def __init__(self):
        self.events: list[tuple[str, str, str]] = []

    async def add_reaction(self, context, message_id, emoji):
        self.events.append(("add", message_id, emoji))
        return True

    async def remove_reaction(self, context, message_id, emoji):
        self.events.append(("remove", message_id, emoji))
        return True


class _RealIndicatorController(_Controller):
    """Wires the REAL ProcessingIndicatorService so the gate exercises the actual
    show_queued/promote path (not a recording stub)."""

    def __init__(self, im: _ReactionIM):
        super().__init__()
        from core.processing_indicator import ProcessingIndicatorService

        self.config = SimpleNamespace(ack_mode="reaction", language="en")
        self._im = im
        self.processing_indicator = ProcessingIndicatorService(self)

    def get_im_client_for_context(self, _context):
        return self._im


def _reaction_request(message: str, message_id: str, runtime_key: str = "session:/repo"):
    from core.processing_indicator import ProcessingIndicatorHandle

    context = SimpleNamespace(
        platform_specific={},
        message_id=message_id,
        platform="slack",
        channel_id="C1",
    )
    handle = ProcessingIndicatorHandle(context=context, reaction_indicator_selected=True)
    return SimpleNamespace(
        context=context,
        message=message,
        composite_session_id=runtime_key,
        processing_indicator=handle,
        subagent_model=None,
        vibe_agent_model="claude-fixture",
        ack_reaction_message_id=None,
        ack_reaction_emoji=None,
        ack_message_id=None,
        typing_indicator_active=False,
        typing_indicator_task=None,
    )


def test_real_indicator_shows_queued_reaction_on_second_message_while_first_holds_gate() -> None:
    """High-fidelity: the REAL ProcessingIndicatorService must put 👌 on the SECOND
    message while the FIRST still holds the runtime gate, then promote it to 👀."""

    async def _run():
        from core.processing_indicator import ACK_REACTION_EMOJI, QUEUED_REACTION_EMOJI

        im = _ReactionIM()
        controller = _RealIndicatorController(im)
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        agent = _RuntimeAgent(release_first)
        service.register(agent)

        req1 = _reaction_request("first", "m1")
        first = asyncio.create_task(service.handle_message("claude", req1))
        for _ in range(200):
            if ("add", "m1", ACK_REACTION_EMOJI) in im.events:
                break
            await asyncio.sleep(0)
        # First (uncontended) has started and shows the running 👀 directly — it must
        # NOT have shown the queued 👌 (no 👌→👀 flash for a non-queued message).
        assert ("add", "m1", ACK_REACTION_EMOJI) in im.events, im.events
        assert ("add", "m1", QUEUED_REACTION_EMOJI) not in im.events, im.events

        req2 = _reaction_request("second", "m2")
        second = asyncio.create_task(service.handle_message("claude", req2))
        for _ in range(200):
            if ("add", "m2", QUEUED_REACTION_EMOJI) in im.events:
                break
            await asyncio.sleep(0)

        # THE KEY ASSERTION: the queued 👌 is on m2 while m1's turn still holds the
        # gate (m2 has not been promoted to 👀 yet).
        assert ("add", "m2", QUEUED_REACTION_EMOJI) in im.events, im.events
        assert ("add", "m2", ACK_REACTION_EMOJI) not in im.events, im.events

        release_first.set()
        service.release_runtime_turn(req1.context)
        await asyncio.wait_for(first, timeout=3)
        await asyncio.wait_for(second, timeout=3)

        # Once m2's turn starts, 👌 is removed and 👀 added on m2.
        assert ("remove", "m2", QUEUED_REACTION_EMOJI) in im.events, im.events
        assert ("add", "m2", ACK_REACTION_EMOJI) in im.events, im.events

    asyncio.run(_run())


class _TypingReactionIM(_ReactionIM):
    """Accepts every typing send, so only the turn can end the keepalive."""

    async def send_typing_indicator(self, _context):
        return True

    async def clear_typing_indicator(self, _context):
        return True


async def _until(predicate) -> None:
    for _ in range(200):
        if predicate():
            return
        await asyncio.sleep(0)
    raise AssertionError("condition never held")


def _spawn_typing_turn(service, controller, message: str, message_id: str):
    """Start a turn as MessageHandler does: eager typing, then the runtime gate,
    both in the turn's own task, which has not run yet on return."""

    request = _reaction_request(message, message_id)
    request.context.thread_id = None

    async def _turn():
        await controller.processing_indicator._start_typing_indicator(request.processing_indicator)
        controller.processing_indicator.apply_to_request(request, request.processing_indicator)
        await service.handle_message("claude", request)

    return request, asyncio.create_task(_turn())


async def _start_typing_turn(service, controller, message: str, message_id: str):
    """Spawn a typing turn and return the request, the turn and its keepalive."""

    request, turn = _spawn_typing_turn(service, controller, message, message_id)
    await _until(lambda: request.processing_indicator.typing_indicator_task is not None)
    return request, turn, request.processing_indicator.typing_indicator_task


async def _cancel(task) -> None:
    task.cancel()
    with pytest.raises(asyncio.CancelledError):
        await task


@pytest.mark.parametrize("queued", [False, True], ids=["uncontended", "queued-reaction"])
def test_cancel_while_waiting_for_backend_finishes_unhanded_typing(queued) -> None:
    """#2164: a turn cancelled in wait_backend_ready was never handed off, so it
    must finish its own indicator. On 74569ae0f only the queued case did."""

    async def _run():
        im = _TypingReactionIM()
        controller = _RealIndicatorController(im)
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        service.register(_RuntimeAgent(release_first))
        first = None
        if queued:
            first, first_turn, _ = await _start_typing_turn(service, controller, "first", "m1")
            await _until(lambda: service._turn_gates["session:/repo"].token)
        service.begin_backend_drain("claude")

        request, turn, typing_task = await _start_typing_turn(service, controller, "second", "m2")
        gate = service._turn_gates["session:/repo"]
        if queued:
            await _until(lambda: ("add", "m2", "👌") in im.events)
            service.release_runtime_turn(first.context)
        await _until(lambda: gate.lock.locked() and not gate.token)

        await _cancel(turn)

        assert typing_task.done()
        assert request.typing_indicator_task is None
        if queued:
            assert ("remove", "m2", "👌") in im.events
            release_first.set()
            await asyncio.wait_for(first_turn, timeout=3)

    asyncio.run(_run())


def test_cancel_while_acquiring_a_just_released_gate_finishes_typing() -> None:
    """#2164: a released lock with a waker still pending reports unlocked, so
    the next turn creates no queued reaction yet still blocks in acquire(). On
    74569ae0f that path finished only a queued reaction."""

    async def _run():
        im = _TypingReactionIM()
        controller = _RealIndicatorController(im)
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        service.register(_RuntimeAgent(release_first))
        first, first_turn, _ = await _start_typing_turn(service, controller, "first", "m1")
        await _until(lambda: service._turn_gates["session:/repo"].token)
        second, second_turn, _ = await _start_typing_turn(service, controller, "second", "m2")
        gate = service._turn_gates["session:/repo"]
        await _until(lambda: ("add", "m2", "👌") in im.events)

        # The third turn is scheduled before the first releases the gate, so it
        # runs before the woken second turn takes the lock: it sees an unlocked
        # gate, creates no queued reaction, and still waits behind the second.
        third, third_turn = _spawn_typing_turn(service, controller, "third", "m3")
        service.release_runtime_turn(first.context)
        await _until(lambda: second.context.platform_specific.get("agent_runtime_turn_token"))
        typing_task = third.processing_indicator.typing_indicator_task
        for _ in range(10):
            await asyncio.sleep(0)
        assert not third_turn.done()
        assert ("add", "m3", "👌") not in im.events

        await _cancel(third_turn)

        assert typing_task.done()
        service.release_runtime_turn(second.context)
        release_first.set()
        await asyncio.wait_for(asyncio.gather(first_turn, second_turn), timeout=3)

    asyncio.run(_run())


def test_released_gate_ends_the_handed_off_typing_keepalive_within_one_tick() -> None:
    """After hand-off the gate token decides whether the turn is current: a
    release that never reached the indicator (a dropped stale emit, a backend
    refresh) still stops typing while the turn's task keeps running."""

    async def _run():
        im = _TypingReactionIM()
        controller = _RealIndicatorController(im)
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        agent = _RuntimeAgent(release_first)
        service.register(agent)
        with patch("core.processing_indicator.TYPING_KEEPALIVE_INTERVAL_SECONDS", 0):
            request, turn, typing_task = await _start_typing_turn(service, controller, "first", "m1")
            await _until(lambda: agent.started == ["first"])
            for _ in range(5):
                await asyncio.sleep(0)
            assert request.processing_indicator.handed_off
            assert not typing_task.done()

            service.release_runtime_turn(request.context)
            await asyncio.sleep(0)
            await asyncio.sleep(0)

            assert typing_task.done()
            assert not turn.done()
            release_first.set()
            await asyncio.wait_for(turn, timeout=3)

    asyncio.run(_run())


def test_processing_indicator_tracked_before_promote() -> None:
    """Regression (Codex P2): the indicator handle must be tracked (registered for
    terminal/cancel cleanup) BEFORE the promote await, so a cancel during promote
    can still find and finish it instead of leaking the queued/fallback indicator."""

    async def _run():
        log: list[str] = []

        class _Indicator:
            def track_turn(self, _context, _request):
                log.append("track")

            async def show_queued_reaction(self, _request):
                return False

            async def promote_reaction_to_running(self, _request, *, agent_name=None):
                log.append("promote")

            async def finish(self, _request_or_handle):
                log.append("finish")

        controller = _Controller()
        controller.processing_indicator = _Indicator()
        service = AgentService(controller=controller)
        controller.agent_service = service
        service.register(_RuntimeAgent())

        request = _request("first")
        request.processing_indicator = object()  # truthy so _track_processing_indicator_turn runs
        await service.handle_message("claude", request)

        assert log[:2] == ["track", "promote"], log

    asyncio.run(_run())


def test_gate_released_when_cancelled_during_promote() -> None:
    """Regression (Codex P1): a cancel after acquire() but during the
    settle/promote awaits must still release the runtime gate, or the token + lock
    leak and later prompts for the same runtime key block forever."""

    async def _run():
        promote_started = asyncio.Event()
        block = asyncio.Event()

        class _BlockingIndicator:
            async def show_queued_reaction(self, _request):
                return False

            async def promote_reaction_to_running(self, _request, *, agent_name=None):
                promote_started.set()
                await block.wait()  # cancel hits here, inside the managed try

            async def finish(self, _request_or_handle):
                return None

        controller = _Controller()
        controller.processing_indicator = _BlockingIndicator()
        controller.emit_agent_message = AsyncMock()  # used by the cancel tidy
        service = AgentService(controller=controller)
        controller.agent_service = service
        service.register(_RuntimeAgent())

        request = _request("first")
        task = asyncio.create_task(service.handle_message("claude", request))
        await asyncio.wait_for(promote_started.wait(), timeout=2)
        gate = service._get_turn_gate("session:/repo")
        assert gate.lock.locked()  # acquired, currently in promote

        task.cancel()
        try:
            await asyncio.wait_for(task, timeout=2)
        except BaseException:
            pass

        # The CancelledError handler schedules a tidy task that releases the gate.
        released = False
        for _ in range(200):
            await asyncio.sleep(0)
            if not gate.lock.locked() and gate.token == "":
                released = True
                break
        assert released  # gate token + lock released despite cancel during promote

    asyncio.run(_run())


def test_queued_reaction_does_not_delay_gate_acquire_fifo() -> None:
    """Regression (Codex P1): a contended turn must reserve its FIFO place on the
    runtime gate WITHOUT waiting for the (network) queued-reaction call. If the 👌
    add were awaited before acquire(), a later same-runtime message could acquire
    first and reorder prompts."""

    async def _run():
        log: list[str] = []
        block = asyncio.Event()

        class _SlowIndicator:
            async def show_queued_reaction(self, _request):
                log.append("show_queued_start")
                await block.wait()  # simulate slow Slack/Feishu reaction API
                return True

            async def promote_reaction_to_running(self, _request, *, agent_name=None):
                log.append("promote")

            async def finish(self, _request_or_handle):
                log.append("finish")

        controller = _Controller()
        controller.processing_indicator = _SlowIndicator()
        service = AgentService(controller=controller)
        controller.agent_service = service
        release_first = asyncio.Event()
        agent = _RuntimeAgent(release_first)
        service.register(agent)

        first_request = _request("first")
        first = asyncio.create_task(service.handle_message("claude", first_request))
        await asyncio.sleep(0)
        assert agent.started == ["first"]
        gate = service._get_turn_gate("session:/repo")

        second = asyncio.create_task(service.handle_message("claude", _request("second")))
        # Let the second turn run far enough to fire the queued-reaction task and
        # reach acquire(). Its show_queued is blocked forever, so if acquire() were
        # gated behind it the gate would have NO waiter.
        entered_queue = False
        for _ in range(200):
            await asyncio.sleep(0)
            if not entered_queue and gate.lock._waiters and len(gate.lock._waiters) >= 1:
                entered_queue = True
            if entered_queue and "show_queued_start" in log:
                break
        # The turn reserved its FIFO place on the gate (it is a waiter) even though
        # its queued reaction is still blocked — proving acquire() was NOT gated
        # behind the reaction network call.
        assert entered_queue
        assert "show_queued_start" in log  # the queued reaction was kicked off concurrently

        # Cleanup: unblock everything and let the tasks settle.
        block.set()
        release_first.set()
        service.release_runtime_turn(first_request.context)
        for task in (first, second):
            try:
                await asyncio.wait_for(task, timeout=3)
            except Exception:
                pass

    asyncio.run(_run())


def test_agent_service_allows_distinct_runtime_keys_in_parallel() -> None:
    async def _run():
        service = AgentService(controller=_Controller())
        agent = _RuntimeAgent()
        service.register(agent)

        await asyncio.gather(
            service.handle_message("claude", _request("one", "one:/repo")),
            service.handle_message("claude", _request("two", "two:/repo")),
        )

        assert sorted(agent.started) == ["one", "two"]

    asyncio.run(_run())


def test_agent_service_falls_back_to_request_runtime_key_for_legacy_agent_stubs() -> None:
    async def _run():
        service = AgentService(controller=_Controller())
        handled = []

        async def _handle_message(request):
            handled.append(request)

        service.register(SimpleNamespace(name="legacy", handle_message=_handle_message))
        request = _request("hello", "legacy:/repo")

        await service.handle_message("legacy", request)

        assert handled == [request]
        assert request.context.platform_specific["agent_runtime_turn_key"] == "legacy:/repo"
        service.release_runtime_turn(request.context)

    asyncio.run(_run())


def test_agent_service_runtime_guard_drops_stale_emits_after_next_turn_starts() -> None:
    async def _run():
        service = AgentService(controller=_Controller())
        agent = _RuntimeAgent()
        service.register(agent)
        first = _request("first")
        second = _request("second")

        await service.handle_message("claude", first)
        assert service.emit_matches_runtime_turn(first.context)
        service.release_runtime_turn(first.context)

        await service.handle_message("claude", second)

        assert service.emit_matches_runtime_turn(second.context)
        assert not service.emit_matches_runtime_turn(first.context)

    asyncio.run(_run())


def test_agent_service_clear_sessions_releases_cleared_runtime_gates() -> None:
    async def _run():
        service = AgentService(controller=_Controller())
        agent = _ClearingRuntimeAgent({"session:/repo", "session:/other"})
        service.register(agent)

        for runtime_key in ("session:/repo", "session:/other", "unrelated:/repo"):
            gate = service._get_turn_gate(runtime_key)
            await gate.lock.acquire()
            gate.token = f"{runtime_key}-token"
            gate.backend = "codex"

        cleared = await service.clear_sessions("scope-1")

        assert cleared == {"codex": 1}
        assert agent.clear_calls == ["scope-1"]
        assert not service._turn_gates["session:/repo"].lock.locked()
        assert not service._turn_gates["session:/other"].lock.locked()
        assert service._turn_gates["unrelated:/repo"].lock.locked()

        service.release_runtime_turn_key("unrelated:/repo")

    asyncio.run(_run())


def test_agent_service_clear_sessions_does_not_release_new_turn_token() -> None:
    async def _run():
        service = AgentService(controller=_Controller())
        runtime_key = "session:/repo"
        gate = service._get_turn_gate(runtime_key)
        await gate.lock.acquire()
        gate.token = "old-token"
        gate.backend = "codex"

        async def _on_clear():
            service.release_runtime_turn_key(runtime_key, "old-token")
            await gate.lock.acquire()
            gate.token = "new-token"
            gate.backend = "codex"

        agent = _ClearingRuntimeAgent({runtime_key}, on_clear=_on_clear)
        service.register(agent)

        cleared = await service.clear_sessions("scope-1")

        assert cleared == {"codex": 1}
        assert service._turn_gates[runtime_key].lock.locked()
        assert service._turn_gates[runtime_key].token == "new-token"

        service.release_runtime_turn_key(runtime_key, "new-token")

    asyncio.run(_run())


def test_agent_service_marks_runtime_started_from_matching_context_only() -> None:
    service = AgentService(controller=_Controller())
    runtime_key = "session:/repo"
    gate = service._get_turn_gate(runtime_key)
    gate.token = "runtime-token"
    gate.backend = "claude"
    gate.request = _request("first")
    gate.agent = SimpleNamespace(record_runtime_turn_start=Mock())
    context = SimpleNamespace(
        platform_specific={
            "agent_runtime_turn_key": runtime_key,
            "agent_runtime_turn_token": "runtime-token",
        }
    )

    service.mark_runtime_turn_started(context)
    service.mark_runtime_turn_started(context)

    assert gate.runtime_started is True
    gate.agent.record_runtime_turn_start.assert_called_once_with(
        runtime_key=runtime_key,
        request=gate.request,
    )

    stale_context = SimpleNamespace(
        platform_specific={
            "agent_runtime_turn_key": runtime_key,
            "agent_runtime_turn_token": "old-token",
        }
    )
    gate.runtime_started = False

    service.mark_runtime_turn_started(stale_context)

    assert gate.runtime_started is False
    gate.agent.record_runtime_turn_start.assert_called_once()


def test_agent_service_contains_terminal_owner_failure_after_releasing_gate() -> None:
    async def _run():
        controller = _Controller()

        class _TurnOwner:
            @staticmethod
            def on_native_terminal(_context, *, outcome):
                assert outcome == "terminal"
                raise RuntimeError("terminal owner write failed")

        controller.session_turns = _TurnOwner()
        service = AgentService(controller=controller)
        runtime_key = "session:/repo"
        gate = service._get_turn_gate(runtime_key)
        await gate.lock.acquire()
        gate.token = "runtime-token"
        context = SimpleNamespace(
            platform_specific={
                "agent_runtime_turn_key": runtime_key,
                "agent_runtime_turn_token": "runtime-token",
            }
        )

        service.release_runtime_turn(context)

        assert not gate.lock.locked()
        assert gate.token == ""

    asyncio.run(_run())


def test_agent_service_clear_backend_sessions_does_not_release_other_backend_gate() -> None:
    async def _run():
        service = AgentService(controller=_Controller())
        runtime_key = "session:/repo"
        gate = service._get_turn_gate(runtime_key)
        await gate.lock.acquire()
        gate.token = "claude-token"
        gate.backend = "claude"
        agent = _ClearingRuntimeAgent({runtime_key})
        service.register(agent)

        count = await service.clear_backend_sessions("codex", "scope-1")

        assert count == 1
        assert gate.lock.locked()
        assert gate.token == "claude-token"
        service.release_runtime_turn_key(runtime_key, "claude-token")

    asyncio.run(_run())


def test_agent_service_refresh_runtime_config_releases_backend_gates() -> None:
    async def _run():
        service = AgentService(controller=_Controller())
        runtime_key = "session:/repo"
        gate = service._get_turn_gate(runtime_key)
        await gate.lock.acquire()
        gate.token = "refresh-token"
        gate.backend = "codex"
        agent = _RefreshingRuntimeAgent({runtime_key})
        service.register(agent)
        runtime_config = object()

        handled = await service.refresh_runtime_config("codex", runtime_config)

        assert handled is True
        assert agent.refresh_calls == [runtime_config]
        assert not gate.lock.locked()
        assert gate.token == ""
        assert gate.backend == ""

    asyncio.run(_run())


@pytest.mark.parametrize("reason", ["not_active", "runtime_unavailable"])
def test_agent_service_releases_runtime_gate_for_stale_stop_without_terminal_evidence(
    reason: str,
) -> None:
    async def _run():
        controller = _Controller()
        terminal_owner = Mock()
        controller.session_turns = SimpleNamespace(on_native_terminal=terminal_owner)
        service = AgentService(controller=controller)
        agent = _StopRuntimeAgent(reason)
        service.register(agent)
        request = _request("stop")
        gate = service._get_turn_gate("session:/repo")
        await gate.lock.acquire()
        gate.token = "stop-token"
        gate.backend = "claude"
        gate.runtime_started = True

        handled = await service.handle_stop("claude", request)

        assert handled is False
        assert not gate.lock.locked()
        assert request.context.platform_specific["agent_runtime_turn_token"] == "stop-token"
        terminal_owner.assert_not_called()

    asyncio.run(_run())


def test_agent_service_keeps_runtime_gate_for_startup_window_stop() -> None:
    async def _run():
        service = AgentService(controller=_Controller())
        agent = _StopRuntimeAgent("not_active")
        service.register(agent)
        request = _request("stop")
        gate = service._get_turn_gate("session:/repo")
        await gate.lock.acquire()
        gate.token = "stop-token"
        gate.backend = "codex"
        gate.runtime_started = False

        handled = await service.handle_stop("claude", request)

        assert handled is False
        assert gate.lock.locked()
        assert gate.token == "stop-token"
        service.release_runtime_turn_key("session:/repo", "stop-token")

    asyncio.run(_run())


def test_agent_service_keeps_runtime_gate_for_interrupt_failure_stop() -> None:
    async def _run():
        service = AgentService(controller=_Controller())
        agent = _StopRuntimeAgent("interrupt_failed")
        service.register(agent)
        request = _request("stop")
        gate = service._get_turn_gate("session:/repo")
        await gate.lock.acquire()
        gate.token = "stop-token"
        gate.backend = "claude"
        gate.runtime_started = True

        handled = await service.handle_stop("claude", request)

        assert handled is False
        assert gate.lock.locked()
        service.release_runtime_turn_key("session:/repo", "stop-token")

    asyncio.run(_run())


def test_agent_service_releases_gate_when_on_running_fails() -> None:
    async def _run():
        controller = _Controller()
        controller.session_turns = _FailingTurnManager()
        service = AgentService(controller=controller)
        agent = _RuntimeAgent()
        service.register(agent)
        request = _request("hello")

        try:
            await service.handle_message("claude", request)
        except RuntimeError as err:
            assert str(err) == "status failed"
        else:
            raise AssertionError("on_running failure should escape")

        gate = service._turn_gates["session:/repo"]
        assert not gate.lock.locked()
        assert gate.token == ""
        assert gate.runtime_started is False
        assert agent.started == []

    asyncio.run(_run())


def test_agent_service_schedules_terminal_tidy_on_cancellation() -> None:
    # C3: a turn cancelled mid-flight (shutdown / SIGTERM) must SCHEDULE a silent
    # terminal result so the outbound chokepoint collapses the stuck status bubble
    # + settles the dot, then re-raise CancelledError.
    async def _run():
        controller = _Controller()
        emit_calls: list[tuple[tuple, dict]] = []
        token_at_emit: list[str] = []
        emitted = asyncio.Event()

        async def _emit(*args, **kwargs):
            # Capture the runtime-turn token AT EMIT TIME. The fix requires the
            # tidy emit to run while the turn is STILL current (token not yet
            # cleared) — otherwise the real result branch drops it as stale and
            # the bubble never collapses. An early release would make this "".
            gate = service._turn_gates.get("session:/repo")
            token_at_emit.append(gate.token if gate else "")
            emit_calls.append((args, kwargs))
            emitted.set()

        controller.emit_agent_message = _emit
        service = AgentService(controller=controller)
        controller.agent_service = service

        hanging = asyncio.Event()

        class _HangingAgent(_RuntimeAgent):
            async def handle_message(self, _request):
                await hanging.wait()  # block until cancelled

        agent = _HangingAgent()
        service.register(agent)
        request = _request("cancel-me")

        task = asyncio.create_task(service.handle_message("claude", request))
        await asyncio.sleep(0.05)  # let it reach the hanging agent
        task.cancel()
        try:
            await task
        except asyncio.CancelledError:
            pass
        else:
            raise AssertionError("CancelledError should propagate")

        # The terminal tidy emit was scheduled (create_task) → let it run.
        await asyncio.wait_for(emitted.wait(), timeout=2.0)
        assert len(emit_calls) == 1
        args, kwargs = emit_calls[0]
        # emit(context, "result", "", is_error=True, level="silent")
        assert args[0] is request.context
        assert args[1] == "result"
        assert kwargs.get("is_error") is True
        assert kwargs.get("level") == "silent"
        # The turn was STILL current at emit time (token not cleared first) so the
        # tidy isn't dropped as stale — this is the ordering the C3 fix guarantees.
        assert token_at_emit and token_at_emit[0]
        # Gate released AFTER the tidy emit so a later prompt can't hang behind the
        # cancelled turn.
        assert not service._turn_gates["session:/repo"].token

    asyncio.run(_run())


def test_message_delivery_023_prewrite_stop_releases_gate_without_terminal_tidy() -> None:
    """MESSAGE-DELIVERY-023: a prewrite Stop cannot retain runtime ownership."""

    async def _run() -> None:
        controller = _Controller()
        stalled_tidy = asyncio.Event()

        async def _stall_terminal_tidy(*_args, **_kwargs) -> None:
            await stalled_tidy.wait()

        controller.emit_agent_message = AsyncMock(side_effect=_stall_terminal_tidy)
        controller.session_turns = Mock()
        surface_cleanup_release = asyncio.Event()

        async def _stall_surface_cleanup(*_args, **_kwargs) -> None:
            await surface_cleanup_release.wait()

        controller.message_dispatcher = SimpleNamespace(
            status_key_for_context=Mock(return_value="status:first"),
            finish_prewrite_stop_surfaces=AsyncMock(side_effect=_stall_surface_cleanup),
            defer_close_after_until_run_terminal=Mock(),
        )
        service = AgentService(controller=controller)
        controller.agent_service = service
        entered = asyncio.Event()

        class _PrewriteAgent(_RuntimeAgent):
            async def handle_message(self, request):
                self.started.append(request.message)
                if request.message == "first":
                    entered.set()
                    await asyncio.Event().wait()

        agent = _PrewriteAgent()
        service.register(agent)
        first = _request("first")
        first.context.platform_specific["close_after"] = True
        first.context.platform_specific["task_execution_id"] = "run-first"
        first_task = asyncio.create_task(service.handle_message("claude", first))
        await asyncio.wait_for(entered.wait(), timeout=0.5)

        mark_prewrite_user_stop(first.context)
        first_task.cancel()
        with pytest.raises(asyncio.CancelledError):
            await first_task
        await asyncio.sleep(0)

        gate = service._turn_gates["session:/repo"]
        assert not gate.lock.locked()
        assert gate.token == ""
        controller.emit_agent_message.assert_not_awaited()
        controller.session_turns.on_native_terminal.assert_not_called()
        controller.message_dispatcher.status_key_for_context.assert_called_once_with(first.context)
        controller.message_dispatcher.finish_prewrite_stop_surfaces.assert_awaited_once_with(
            first.context,
            consolidated_key="status:first",
        )
        controller.message_dispatcher.defer_close_after_until_run_terminal.assert_called_once_with(
            first.context
        )

        second = _request("second")
        await asyncio.wait_for(service.handle_message("claude", second), timeout=0.5)
        assert agent.started == ["first", "second"]
        service.release_runtime_turn(second.context)
        surface_cleanup_release.set()
        await asyncio.sleep(0)

    asyncio.run(_run())


def test_agent_service_releases_gate_when_exception_terminal_emit_fails() -> None:
    """HFR-001: pre-accept failure releases ownership even if terminal emit fails."""

    async def _run():
        controller = _Controller()

        async def _emit(*_args, **_kwargs):
            raise RuntimeError("send failed")

        controller.emit_agent_message = _emit
        service = AgentService(controller=controller)
        agent = _RaisingRuntimeAgent()
        service.register(agent)
        request = _request("boom")

        try:
            await service.handle_message("claude", request)
        except RuntimeError as err:
            assert str(err) == "backend failed"
        else:
            raise AssertionError("backend exception should escape")

        assert not service._turn_gates["session:/repo"].lock.locked()

    asyncio.run(_run())


def test_agent_service_reports_backend_failure_before_releasing_runtime_turn() -> None:
    """A caller-owned failure receipt must reach settlement before Turn release."""

    async def _run() -> None:
        from core.backend_failure import emit_backend_failure

        controller = _Controller()
        service = AgentService(controller=controller)
        controller.agent_service = service
        service.register(_RaisingRuntimeAgent())
        request = _request("boom")
        request.context.platform = "slack"
        request.context.platform_specific = {
            "turn_token": "turn-backend-failure",
            "task_execution_id": "run-backend-failure",
            "task_trigger_kind": "watch",
        }
        emitted: list[tuple[str, object, str]] = []

        async def _emit(_context, message_type, _text, **kwargs):
            gate = service._turn_gates["session:/repo"]
            emitted.append((message_type, kwargs.get("output"), gate.token))
            delivery = kwargs.get("delivery")
            if message_type == "notify" and delivery is not None:
                delivery.send_returned = True
                delivery.delivered_id = "slack-message-1"
                return delivery.delivered_id
            return None

        async def _report(error: BaseException) -> None:
            await emit_backend_failure(
                controller,
                request.context,
                "claude",
                str(error),
                display_text=f"Error: {error}",
                request=request,
            )

        controller.emit_agent_message = _emit
        request.failure_handler = _report

        with pytest.raises(RuntimeError, match="backend failed"):
            await service.handle_message("claude", request)

        assert [message_type for message_type, _output, _token in emitted] == [
            "notify",
            "result",
        ]
        terminal = emitted[-1][1]
        assert terminal.metadata["turn_failure_notification"] == {
            "failure_id": "turn:turn-backend-failure",
            "ack_evidence": "delivery_only",
            "delivered": True,
        }
        assert all(token for _message_type, _output, token in emitted)
        assert request.failure_handled is True
        assert not service._turn_gates["session:/repo"].lock.locked()

    asyncio.run(_run())


def test_agent_service_recovers_accepted_turn_when_owned_backend_dies() -> None:
    """A dead accepted backend must release the runtime FIFO without a timeout."""

    async def _run() -> None:
        controller = _Controller()
        recovered = asyncio.Event()
        service = AgentService(controller=controller)
        controller.agent_service = service

        class _AcceptedThenDeadAgent(_RuntimeAgent):
            def __init__(self) -> None:
                super().__init__()
                self.alive = True

            async def handle_message(self, request) -> None:
                self.started.append(request.message)
                service.mark_runtime_turn_started(request.context)

            def backend_alive(self, _context):
                return self.alive

        agent = _AcceptedThenDeadAgent()
        service.register(agent)
        service._liveness_probe_interval_seconds = 0.01
        service._liveness_failure_grace_seconds = 0.02

        async def _emit(context, message_type, text, **kwargs):
            assert message_type == "result"
            assert text == ""
            assert kwargs["is_error"] is True
            assert kwargs["level"] == "silent"
            assert kwargs["terminal_error"] == "backend_runtime_exited_before_terminal"
            service.release_runtime_turn(context)
            recovered.set()

        controller.emit_agent_message = _emit

        first = _request("first")
        second = _request("second")
        await service.handle_message("claude", first)
        assert service._turn_gates["session:/repo"].lock.locked()

        successor = asyncio.create_task(service.handle_message("claude", second))
        await asyncio.sleep(0)
        assert agent.started == ["first"]

        agent.alive = False
        try:
            await asyncio.wait_for(recovered.wait(), timeout=0.5)
            await asyncio.wait_for(successor, timeout=0.5)
            assert agent.started == ["first", "second"]
        finally:
            if not successor.done():
                successor.cancel()
                await asyncio.gather(successor, return_exceptions=True)
            service.release_runtime_turn(second.context)

    asyncio.run(_run())


def test_hfr_432_opencode_timeout_releases_fifo_and_shared_runtime() -> None:
    """HFR-432: one stuck native Turn cannot retain either OpenCode lane forever."""

    async def _run() -> None:
        controller = _Controller()
        service = AgentService(controller=controller)
        controller.agent_service = service
        emitted: list[tuple[str, str, bool, str | None]] = []

        class _Server:
            def __init__(self) -> None:
                self.poll_started = asyncio.Event()
                self.aborted = asyncio.Event()
                self.abort_calls: list[tuple[str, str]] = []

            async def list_messages(self, session_id: str, directory: str):
                self.poll_started.set()
                return [
                    {
                        "info": {
                            "id": "msg-stuck",
                            "role": "assistant",
                            "time": {},
                        },
                        "parts": [],
                    }
                ]

            async def abort_session(self, session_id: str, directory: str) -> bool:
                self.abort_calls.append((session_id, directory))
                self.aborted.set()
                return True

        server = _Server()

        class _OpenCodeTimeoutAgent(_RuntimeAgent):
            name = "opencode"
            opencode_config = SimpleNamespace(
                error_retry_limit=0,
                active_turn_timeout_seconds=0.05,
            )

            def __init__(self) -> None:
                super().__init__()
                self.poll_loop = OpenCodePollLoop(self)
                self.other_finished = asyncio.Event()

            @staticmethod
            def _extract_response_text(_message) -> str:
                return ""

            @staticmethod
            def _to_relative_path(path: str, _working_path: str) -> str:
                return path

            async def record_model_hub_native_failure(self, _context, _diagnostic):
                return False

            async def handle_message(self, request) -> None:
                self.started.append(request.message)
                service.mark_runtime_turn_started(request.context)
                if request.message == "first":
                    final_text, should_emit = await self.poll_loop.run_prompt_poll(
                        request,
                        server,
                        "oc-stuck",
                        agent_to_use=None,
                        model_dict=None,
                        reasoning_effort=None,
                        baseline_message_ids=set(),
                    )
                    assert final_text is None
                    assert should_emit is False
                    return
                if request.message == "other-agent":
                    # Different runtime key, but the shared provider lane cannot
                    # progress until the stuck native Turn is actually aborted.
                    await server.aborted.wait()
                    self.other_finished.set()
                await controller.emit_agent_message(
                    request.context,
                    "result",
                    f"{request.message} complete",
                )

        agent = _OpenCodeTimeoutAgent()
        agent.controller = controller
        service.register(agent)
        controller._t = lambda key, **kwargs: f"{key}:{kwargs.get('seconds')}"

        async def _emit(context, message_type, text, **kwargs):
            emitted.append(
                (
                    message_type,
                    text,
                    bool(kwargs.get("is_error")),
                    kwargs.get("terminal_error"),
                )
            )
            if message_type == "result":
                service.release_runtime_turn(context)

        controller.emit_agent_message = _emit

        def _opencode_request(message: str, runtime_key: str, run_id: str) -> AgentRequest:
            return AgentRequest(
                context=MessageContext(
                    user_id=run_id,
                    channel_id=run_id,
                    platform="avibe",
                    platform_specific={"task_execution_id": run_id},
                ),
                message=message,
                user_message=message,
                working_path="/tmp/work",
                base_session_id=runtime_key,
                composite_session_id=runtime_key,
                session_key=f"avibe::{runtime_key}",
                vibe_agent_model="fixture-provider/fixture-model",
            )

        first = asyncio.create_task(
            service.handle_message(
                "opencode",
                _opencode_request("first", "agent-a:/tmp/work", "run-first"),
            )
        )
        await asyncio.wait_for(server.poll_started.wait(), timeout=0.5)
        same_session_successor = asyncio.create_task(
            service.handle_message(
                "opencode",
                _opencode_request(
                    "same-session-successor",
                    "agent-a:/tmp/work",
                    "run-successor",
                ),
            )
        )
        other_agent = asyncio.create_task(
            service.handle_message(
                "opencode",
                _opencode_request("other-agent", "agent-b:/tmp/work", "run-other"),
            )
        )

        await asyncio.sleep(0)
        assert "same-session-successor" not in agent.started
        assert "other-agent" in agent.started

        await asyncio.wait_for(
            asyncio.gather(first, same_session_successor, other_agent),
            timeout=1,
        )

        assert server.abort_calls == [("oc-stuck", "/tmp/work")]
        assert agent.other_finished.is_set()
        assert agent.started.index("same-session-successor") > agent.started.index("first")
        timeout_results = [item for item in emitted if item[2]]
        assert timeout_results == [
            (
                "result",
                "",
                True,
                "OpenCode active turn exceeded the configured 0.05-second wall-clock limit",
            )
        ]
        assert all("No response from OpenCode" not in item[1] for item in emitted)

    asyncio.run(_run())


def test_agent_service_reports_captured_resource_failure_after_turn_acceptance() -> None:
    async def _run() -> None:
        controller = _Controller()
        recovered = asyncio.Event()
        service = AgentService(controller=controller)
        controller.agent_service = service

        class _AcceptedThenDeadAgent(_RuntimeAgent):
            name = "codex"

            def __init__(self) -> None:
                super().__init__()
                self.current = SimpleNamespace(alive=True)

            async def handle_message(self, request) -> None:
                service.mark_runtime_turn_started(request.context)

            def capture_backend_liveness(self, _context):
                accepted = self.current
                return lambda: accepted.alive

            def capture_backend_exit_failure(self, _context):
                accepted = self.current
                return lambda: (
                    ("backend_runtime_exited_before_terminal\nResource diagnosis: pids", "❌ pids limit")
                    if not accepted.alive
                    else None
                )

        agent = _AcceptedThenDeadAgent()
        service.register(agent)
        service._liveness_probe_interval_seconds = 0.005
        service._liveness_failure_grace_seconds = 0.005
        emitted: list[tuple[str, str, object]] = []

        async def _emit(context, message_type, text, **kwargs):
            emitted.append((message_type, text, kwargs.get("terminal_error")))
            if message_type == "result":
                service.release_runtime_turn(context)
                recovered.set()

        controller.emit_agent_message = _emit
        request = _request("long-running codex turn")
        await service.handle_message("codex", request)
        accepted = agent.current
        agent.current = SimpleNamespace(alive=True)
        accepted.alive = False

        await asyncio.wait_for(recovered.wait(), timeout=0.5)
        assert [message_type for message_type, _, _ in emitted] == ["notify", "result"]
        assert emitted[0][1] == "❌ pids limit"
        assert "Resource diagnosis: pids" in emitted[1][2]
        assert not service._turn_gates["session:/repo"].lock.locked()

    asyncio.run(_run())


def test_agent_service_recovery_uses_the_accepted_backend_generation() -> None:
    """HFR-002: a replacement runtime cannot hide the accepted owner's death."""

    async def _run() -> None:
        controller = _Controller()
        recovered = asyncio.Event()
        service = AgentService(controller=controller)
        controller.agent_service = service

        class _GenerationAgent(_RuntimeAgent):
            def __init__(self) -> None:
                super().__init__()
                self.current = SimpleNamespace(alive=True)

            async def handle_message(self, request) -> None:
                self.started.append(request.message)
                service.mark_runtime_turn_started(request.context)

            def backend_alive(self, _context):
                return self.current.alive

            def capture_backend_liveness(self, _context):
                accepted = self.current
                return lambda: accepted.alive

        agent = _GenerationAgent()
        service.register(agent)
        service._liveness_probe_interval_seconds = 0.005
        service._liveness_failure_grace_seconds = 0.005

        async def _emit(context, *_args, **_kwargs):
            service.release_runtime_turn(context)
            recovered.set()

        controller.emit_agent_message = _emit
        first = _request("first")
        second = _request("second")
        await service.handle_message("claude", first)
        accepted = agent.current
        successor = asyncio.create_task(service.handle_message("claude", second))
        await asyncio.sleep(0)

        agent.current = SimpleNamespace(alive=True)
        accepted.alive = False
        try:
            await asyncio.wait_for(recovered.wait(), timeout=0.5)
            await asyncio.wait_for(successor, timeout=0.5)
            assert agent.started == ["first", "second"]
            assert service.backend_alive(second.context) is True
        finally:
            if not successor.done():
                successor.cancel()
                await asyncio.gather(successor, return_exceptions=True)
            service.release_runtime_turn(second.context)

    asyncio.run(_run())


@pytest.mark.parametrize("liveness", [True, None])
def test_agent_service_never_steals_live_or_unknown_long_running_owner(liveness) -> None:
    """HFR-005/HFR-006: age alone never releases a live or unknown owner."""

    async def _run() -> None:
        controller = _Controller()
        service = AgentService(controller=controller)
        controller.agent_service = service
        controller.emit_agent_message = AsyncMock(side_effect=AssertionError("live owner was stolen"))

        class _LongRunningAgent(_RuntimeAgent):
            async def handle_message(self, request) -> None:
                self.started.append(request.message)
                service.mark_runtime_turn_started(request.context)

            def backend_alive(self, _context):
                return liveness

        agent = _LongRunningAgent()
        service.register(agent)
        service._liveness_probe_interval_seconds = 0.005
        service._liveness_failure_grace_seconds = 0.005

        first = _request("first")
        second = _request("second")
        await service.handle_message("claude", first)
        successor = asyncio.create_task(service.handle_message("claude", second))
        try:
            await asyncio.sleep(0.06)
            assert agent.started == ["first"]
            assert not successor.done()
            controller.emit_agent_message.assert_not_awaited()

            service.release_runtime_turn(first.context)
            await asyncio.wait_for(successor, timeout=0.5)
            assert agent.started == ["first", "second"]
        finally:
            if not successor.done():
                successor.cancel()
                await asyncio.gather(successor, return_exceptions=True)
            service.release_runtime_turn(second.context)

    asyncio.run(_run())


def test_agent_service_requires_dead_backend_after_grace_before_recovery() -> None:
    """HFR-006: a transient false probe cannot release the accepted owner."""

    async def _run() -> None:
        controller = _Controller()
        service = AgentService(controller=controller)
        controller.agent_service = service
        controller.emit_agent_message = AsyncMock(side_effect=AssertionError("false blip stole the owner"))

        class _FlappingAgent(_RuntimeAgent):
            alive = True

            async def handle_message(self, request) -> None:
                self.started.append(request.message)
                service.mark_runtime_turn_started(request.context)

            def backend_alive(self, _context):
                return self.alive

        agent = _FlappingAgent()
        service.register(agent)
        service._liveness_probe_interval_seconds = 0.005
        service._liveness_failure_grace_seconds = 0.04

        request = _request("first")
        await service.handle_message("claude", request)
        agent.alive = False
        await asyncio.sleep(0.015)
        agent.alive = True
        await asyncio.sleep(0.06)

        assert service._turn_gates["session:/repo"].lock.locked()
        assert service.runtime_turn_started(request.context) is True
        controller.emit_agent_message.assert_not_awaited()
        service.release_runtime_turn(request.context)

    asyncio.run(_run())


def test_late_terminal_from_recovered_owner_cannot_release_successor() -> None:
    """HFR-007: token identity makes dead-owner recovery and late results idempotent."""

    async def _run() -> None:
        controller = _Controller()
        recovered = asyncio.Event()
        service = AgentService(controller=controller)
        controller.agent_service = service

        class _RecoveringAgent(_RuntimeAgent):
            alive = True

            async def handle_message(self, request) -> None:
                self.started.append(request.message)
                service.mark_runtime_turn_started(request.context)

            def backend_alive(self, _context):
                return self.alive

        agent = _RecoveringAgent()
        service.register(agent)
        service._liveness_probe_interval_seconds = 0.005
        service._liveness_failure_grace_seconds = 0.005
        terminal_calls = 0

        async def _emit(context, *_args, **_kwargs):
            nonlocal terminal_calls
            terminal_calls += 1
            service.release_runtime_turn(context)
            recovered.set()

        controller.emit_agent_message = _emit
        first = _request("first")
        second = _request("second")
        await service.handle_message("claude", first)
        first_token = first.context.platform_specific["agent_runtime_turn_token"]
        successor = asyncio.create_task(service.handle_message("claude", second))
        agent.alive = False
        try:
            await asyncio.wait_for(recovered.wait(), timeout=0.5)
            agent.alive = True
            await asyncio.wait_for(successor, timeout=0.5)
            gate = service._turn_gates["session:/repo"]
            successor_token = gate.token
            assert successor_token and successor_token != first_token

            # The old context still carries the old token. A delayed result's
            # normal release hook must not touch the successor owner.
            service.release_runtime_turn(first.context)
            assert gate.token == successor_token
            assert gate.lock.locked()
            assert terminal_calls == 1
        finally:
            if not successor.done():
                successor.cancel()
                await asyncio.gather(successor, return_exceptions=True)
            service.release_runtime_turn(second.context)

    asyncio.run(_run())


def test_real_child_exit_after_acceptance_recovers_runtime_fifo() -> None:
    """HFR-002: an owned OS child exit drives the same shared recovery path."""

    async def _run() -> None:
        process = await asyncio.create_subprocess_exec(
            sys.executable,
            "-c",
            "import time; time.sleep(60)",
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
        )
        transport = CodexTransport(binary=sys.executable, cwd="/")
        transport._process = process
        transport._reader_task = asyncio.create_task(transport._reader_loop())

        controller = _Controller()
        recovered = asyncio.Event()
        service = AgentService(controller=controller)
        controller.agent_service = service

        class _ProcessBackedAgent(_RuntimeAgent):
            name = "codex"

            async def handle_message(self, request) -> None:
                self.started.append(request.message)
                service.mark_runtime_turn_started(request.context)

            def backend_alive(self, _context):
                return transport.is_alive

        agent = _ProcessBackedAgent()
        service.register(agent)
        service._liveness_probe_interval_seconds = 0.005
        service._liveness_failure_grace_seconds = 0.005
        terminal_calls = 0

        async def _emit(context, *_args, **_kwargs):
            nonlocal terminal_calls
            terminal_calls += 1
            service.release_runtime_turn(context)
            recovered.set()

        controller.emit_agent_message = _emit
        first = _request("first", runtime_key="codex:/repo")
        second = _request("second", runtime_key="codex:/repo")
        successor = None
        try:
            await service.handle_message("codex", first)
            successor = asyncio.create_task(service.handle_message("codex", second))
            process.kill()
            await process.wait()
            await transport._reader_task

            await asyncio.wait_for(recovered.wait(), timeout=0.5)
            await asyncio.wait_for(successor, timeout=0.5)
            assert agent.started == ["first", "second"]
            assert terminal_calls == 1
        finally:
            if process.returncode is None:
                process.kill()
                await process.wait()
            if successor is not None and not successor.done():
                successor.cancel()
                await asyncio.gather(successor, return_exceptions=True)
            service.release_runtime_turn(second.context)

    asyncio.run(_run())


def test_a_failed_teardown_is_retried_until_it_finishes() -> None:
    """A disabled backend's processes stop eventually even if the first stop fails.

    The agent is already out of routing, so nothing else would ever stop them.
    """

    async def run() -> None:
        attempts: list[int] = []

        async def stop() -> None:
            attempts.append(len(attempts))
            if len(attempts) < 3:
                raise RuntimeError("process survived its stop")

        service = AgentService(controller=SimpleNamespace())
        assert await service.run_until_done("disabled:codex", stop) is False
        await service.retry_pending()
        assert len(attempts) == 2
        await service.retry_pending()
        assert len(attempts) == 3
        await service.retry_pending()
        assert len(attempts) == 3

    asyncio.run(run())