Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

208 KB
Raw Blame
from __future__ import annotations

import asyncio
import json
import logging
import os
import sys
import threading
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from config import paths
from core import command_runner, watch_worker, watches as watches_module
from core.process_isolation import (
    PersistedProcessIdentity,
    ProcessIdentity,
    fingerprint_process_marker,
)
from core.runtime_work import RuntimeWorkItem, RuntimeWorkLane, RuntimeWorkSupervisor
from core.scheduled_tasks import TaskExecutionStore
from core.watches import (
    CIRCUIT_BREAKER_METADATA_KEY,
    DELIVERY_ACK_METADATA_KEY,
    FOLLOW_UP_RUN_ID_METADATA_KEY,
    LAST_DELIVERY_ENV,
    LIFETIME_STARTED_AT_METADATA_KEY,
    NO_EVENT_EXIT_CODE,
    NO_EVENT_MARKER,
    RECENT_EVENT_TIMESTAMPS_METADATA_KEY,
    WATCH_CIRCUIT_OUTPUT_LIMIT,
    WATCH_ID_ENV,
    ManagedWatchService,
    ManagedWatchStore,
    WatchRuntimeStateStore,
    _CycleResult,
    _cycle_env,
    _ManagedWatchRuntimeWorkHandler,
    _recent_event_timestamps,
    _StaleWorkerRecovery,
)
from storage.background import (
    WATCH_HOOK_OUTCOME_CIRCUIT_REPAIR,
    WATCH_HOOK_OUTCOME_EVENT,
    WATCH_HOOK_OUTCOME_METADATA_KEY,
    WATCH_HOOK_OUTCOME_WAITER_FAILURE,
    SQLiteBackgroundTaskStore,
)
from tests.fake_pid_helpers import fake_pid

# Pids no process can hold: a supervisor that never ran, and a recorded watch worker.
_SUPERVISOR_PID = fake_pid(0)
_WORKER_PID = fake_pid(1)

TEST_MARKER = "test-watch-worker"
TEST_FINGERPRINT = fingerprint_process_marker(TEST_MARKER)


class _ControlledWatchClock:
    """Clock double for the wall-time cooldown branch in the real wait loop."""

    def __init__(self, current: datetime) -> None:
        self.current = current

    def now(self, tz=None) -> datetime:
        if tz is None:
            return self.current.replace(tzinfo=None)
        return self.current.astimezone(tz)

    def fromisoformat(self, value: str) -> datetime:
        return datetime.fromisoformat(value)


def _quiet_waiter_command(summary: str = "") -> list[str]:
    """A waiter that ends a cycle the way the no-event protocol asks it to.

    The marker is half the signal: the exit code on its own is ``sysexits`` EX_USAGE
    and stays a failure.
    """

    script = "import sys; "
    if summary:
        script += f"print({summary!r}, file=sys.stderr); "
    script += f"print({NO_EVENT_MARKER!r}, file=sys.stderr); sys.exit({NO_EVENT_EXIT_CODE})"
    return [sys.executable, "-c", script]


class _FakeStdin:
    def __init__(self) -> None:
        self.payload = bytearray()
        self.closed = False

    def write(self, payload: bytes) -> None:
        self.payload.extend(payload)

    async def drain(self) -> None:
        return None

    def close(self) -> None:
        self.closed = True


class _BrokenStdin(_FakeStdin):
    async def drain(self) -> None:
        raise BrokenPipeError


class _FakeProcess:
    def __init__(
        self,
        *,
        stdin: _FakeStdin | None = None,
        stderr: bytes = b"",
    ) -> None:
        self.pid = _SUPERVISOR_PID
        self.returncode = 0
        self.stdin = stdin or _FakeStdin()
        self.stderr = stderr

    async def communicate(self) -> tuple[bytes, bytes]:
        return b"ok\n", self.stderr


async def _start_watch_service(service: ManagedWatchService) -> None:
    service.start()
    startup_task = service._startup_task
    if startup_task is not None:
        await startup_task


async def _await_backed_off_watch(
    service: ManagedWatchService,
    watch_id: str,
    timeout: float = 10.0,
) -> None:
    """Wait for a watch to back off instead of sleeping a fixed interval.

    Backing off needs a real waiter subprocess to start and its result to reach
    the raising store, so the wait covers a process spawn plus interpreter
    startup. A fixed 0.12s was enough locally and not on a loaded CI runner.
    """

    deadline = time.monotonic() + timeout
    while watch_id not in service._watch_store_retries:
        assert time.monotonic() < deadline, "watch never backed off after the store error"
        await asyncio.sleep(0.02)


async def _await_condition(predicate, message: str, timeout: float = 10.0) -> None:
    deadline = time.monotonic() + timeout
    while not predicate():
        assert time.monotonic() < deadline, message
        await asyncio.sleep(0.02)


def _add_recovery_watch(
    store: ManagedWatchStore,
    *,
    command: list[str],
    shell_command: str | None = None,
):
    return store.add_watch(
        name="Recovery watch",
        session_key="slack::channel::C123",
        command=command,
        shell_command=shell_command,
        prefix=None,
        cwd=None,
        mode="forever",
        timeout_seconds=0,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )


def _record_watch_pid(
    runtime_store: WatchRuntimeStateStore,
    watch_id: str,
    pid: int,
    *,
    identity: PersistedProcessIdentity | None = None,
) -> None:
    entry = {
        "running": True,
        "pid": pid,
        "started_at": "2026-05-15T00:00:00+00:00",
        "updated_at": "2026-05-15T00:00:01+00:00",
    }
    if identity is not None:
        entry["process_identity"] = {
            "pid": identity.pid,
            "create_time": identity.create_time,
            "worker_fingerprint": identity.worker_fingerprint,
        }
    runtime_store.write(
        {
            "watches": {
                watch_id: entry,
            }
        }
    )


def _live_identity(
    *,
    pid: int = _WORKER_PID,
    create_time: float = 123.0,
    worker_fingerprint: str | None = TEST_FINGERPRINT,
) -> ProcessIdentity:
    return ProcessIdentity(
        pid=pid,
        create_time=create_time,
        worker_fingerprint=worker_fingerprint,
    )


def _persisted_identity(
    *,
    pid: int = _WORKER_PID,
    create_time: float = 123.0,
    worker_fingerprint: str = TEST_FINGERPRINT,
) -> PersistedProcessIdentity:
    return PersistedProcessIdentity(
        pid=pid,
        create_time=create_time,
        worker_fingerprint=worker_fingerprint,
    )


def test_managed_watch_store_round_trip(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    watch = store.add_watch(
        name="Watch CI",
        session_key="slack::channel::C123",
        command=["python3", "wait.py"],
        shell_command=None,
        prefix="CI finished.",
        cwd="/tmp",
        mode="forever",
        timeout_seconds=600,
        lifetime_timeout_seconds=3600,
        retry_exit_codes=[75],
        retry_delay_seconds=45,
        post_to="channel",
        deliver_key=None,
    )

    payload = json.loads(store.path.read_text(encoding="utf-8"))
    reloaded = ManagedWatchStore(store.path)
    saved = reloaded.get_watch(watch.id)

    assert payload["watches"][0]["id"] == watch.id
    assert saved is not None
    assert saved.name == "Watch CI"
    assert saved.mode == "forever"
    assert saved.retry_exit_codes == [75]
    assert saved.post_to == "channel"


def test_hfr_174_watch_definition_commit_emits_watch_hint(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """HFR-174: task and watch definition commits wake separate reconcile owners."""

    from core import inbox_events

    monkeypatch.setattr(inbox_events, "_CONTROLLER_PROCESS", True)
    events: list[tuple[str, dict]] = []
    subscription = inbox_events.bus.subscribe_callback(
        lambda event_type, payload: events.append((event_type, payload))
    )
    store = ManagedWatchStore(tmp_path / "watches.json")
    store.add_watch(
        name="Definition wake",
        session_key="avibe::agent::default",
        command=["true"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="once",
        timeout_seconds=30,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )
    inbox_events.bus.unsubscribe(subscription)

    assert events == [
        (inbox_events.DEFINITIONS_UPDATED_EVENT, {"definition_type": "watch"})
    ]


def test_managed_watch_store_preserves_zero_values_on_reload(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    watch = store.add_watch(
        name="Watch Zero",
        session_key="slack::channel::C123",
        command=["python3", "wait.py"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="forever",
        timeout_seconds=0,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )

    reloaded = ManagedWatchStore(store.path)
    saved = reloaded.get_watch(watch.id)

    assert saved is not None
    assert saved.timeout_seconds == 0
    assert saved.lifetime_timeout_seconds == 0
    assert saved.retry_delay_seconds == 0


def test_managed_watch_store_recovery_accepts_empty_command_arguments(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    watch = _add_recovery_watch(store, command=[sys.executable, "wait.py", ""])

    recovered = store.list_watches_for_recovery()

    assert [item.id for item in recovered] == [watch.id]
    assert recovered[0].command == [sys.executable, "wait.py", ""]


def test_malformed_remote_context_disables_watch_before_waiter_spawn(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    watch = store.add_watch(
        name="Legacy remote watch",
        session_key="slack::channel::C123",
        command=[sys.executable, "-c", "raise AssertionError('must not run')"],
        shell_command=None,
        prefix=None,
        cwd=str(tmp_path),
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )
    watch.metadata = {"resource_user_context": {"sub": "legacy-remote-user"}}
    store.upsert_watch(watch)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    service._running = True
    service._requires_service_lease = False

    async def unexpected_run_cycle(*_args, **_kwargs):
        raise AssertionError("remote-origin waiter must not spawn")

    service._run_cycle = unexpected_run_cycle  # type: ignore[method-assign]
    asyncio.run(service._run_watch(watch.id))

    saved = store.get_watch(watch.id)
    assert saved is not None
    assert saved.enabled is False
    assert saved.last_started_at is None
    assert saved.last_error == "harness_access_forbidden"


def test_managed_watch_exec_uses_stable_supervisor(tmp_path: Path, monkeypatch) -> None:
    captured: dict[str, object] = {}
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    watch = store.add_watch(
        name="Watch Python",
        session_key="slack::channel::C123",
        command=["python3", "-c", "print('ok')"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )

    async def fake_create_subprocess_exec(*args, **kwargs):
        captured["args"] = args
        captured["kwargs"] = kwargs
        process = _FakeProcess()
        captured["process"] = process
        return process

    monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_create_subprocess_exec)

    result = asyncio.run(service._run_cycle(watch, timeout_seconds=5))

    assert result.exit_code == 0
    assert captured["args"] == (os.path.abspath(sys.executable), command_runner.supervisor_path())
    assert captured["kwargs"]["stdin"] == asyncio.subprocess.PIPE
    assert captured["kwargs"]["stdout"] == asyncio.subprocess.PIPE
    assert captured["kwargs"]["stderr"] == asyncio.subprocess.PIPE
    assert captured["kwargs"]["cwd"] == str(paths.get_vibe_remote_dir())
    process = captured["process"]
    assert isinstance(process, _FakeProcess)
    assert process.stdin.closed is True
    assert json.loads(process.stdin.payload) == {
        "version": 1,
        "command": ["python3", "-c", "print('ok')"],
        "shell_command": None,
    }


def test_managed_watch_shell_uses_stable_supervisor(tmp_path: Path, monkeypatch) -> None:
    captured: dict[str, object] = {}
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    watch = store.add_watch(
        name="Watch Shell",
        session_key="slack::channel::C123",
        command=[],
        shell_command="python3 -c 'print(\"ok\")'",
        prefix=None,
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )

    async def fake_create_subprocess_exec(*args, **kwargs):
        captured["args"] = args
        captured["kwargs"] = kwargs
        process = _FakeProcess()
        captured["process"] = process
        return process

    monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_create_subprocess_exec)

    result = asyncio.run(service._run_cycle(watch, timeout_seconds=5))

    assert result.exit_code == 0
    assert captured["args"] == (os.path.abspath(sys.executable), command_runner.supervisor_path())
    assert captured["kwargs"]["stdin"] == asyncio.subprocess.PIPE
    assert captured["kwargs"]["stdout"] == asyncio.subprocess.PIPE
    assert captured["kwargs"]["stderr"] == asyncio.subprocess.PIPE
    assert captured["kwargs"]["cwd"] == str(paths.get_vibe_remote_dir())
    process = captured["process"]
    assert isinstance(process, _FakeProcess)
    assert process.stdin.closed is True
    assert json.loads(process.stdin.payload) == {
        "version": 1,
        "command": [],
        "shell_command": "python3 -c 'print(\"ok\")'",
    }


def test_managed_watch_clears_supervisor_state_when_startup_pipe_breaks(tmp_path: Path, monkeypatch) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    watch = store.add_watch(
        name="Broken supervisor",
        session_key="slack::channel::C123",
        command=["python3", "-c", "print('ok')"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )
    process = _FakeProcess(stdin=_BrokenStdin())

    async def fake_create_subprocess_exec(*_args, **_kwargs):
        return process

    monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_create_subprocess_exec)

    with pytest.raises(RuntimeError, match="Watch worker supervisor exited during startup"):
        asyncio.run(service._run_cycle(watch, timeout_seconds=5))

    assert process.stdin.closed is True
    assert service._active_pids == {}
    assert service._active_process_identities == {}


def test_managed_watch_localizes_supervisor_startup_failure(tmp_path: Path, monkeypatch) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    service = ManagedWatchService(
        controller=SimpleNamespace(config=SimpleNamespace(language="zh")),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    watch = store.add_watch(
        name="Broken supervisor",
        session_key="slack::channel::C123",
        command=["python3", "-c", "print('ok')"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )
    process = _FakeProcess(
        stdin=_BrokenStdin(),
        stderr=watch_worker.encode_watch_worker_error("invalidCommand").encode(),
    )

    async def fake_create_subprocess_exec(*_args, **_kwargs):
        return process

    monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_create_subprocess_exec)

    with pytest.raises(RuntimeError, match="Watch 监控进程失败:Watch 工作进程命令无效。"):
        asyncio.run(service._run_cycle(watch, timeout_seconds=5))


def test_managed_watch_legacy_none_cwd_survives_deleted_process_cwd(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    watch = store.add_watch(
        name="Legacy watch",
        session_key="slack::channel::C123",
        command=[sys.executable, "-c", "import os; print(os.getcwd())"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )
    deleted_cwd = tmp_path / "deleted-service-cwd"
    deleted_cwd.mkdir()
    original_cwd = Path.cwd()

    os.chdir(deleted_cwd)
    deleted_cwd.rmdir()
    try:
        result = asyncio.run(service._run_cycle(watch, timeout_seconds=5))
    finally:
        os.chdir(original_cwd)

    assert result.exit_code == 0
    assert result.stdout == str(paths.get_vibe_remote_dir())


def test_managed_watch_store_uses_sqlite_when_path_is_default(tmp_path: Path, monkeypatch) -> None:
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    store = ManagedWatchStore()
    watch = store.add_watch(
        name="Watch CI",
        session_key="slack::channel::C123",
        session_id="sesk8m4q2p7x",
        command=["python3", "wait.py"],
        shell_command=None,
        prefix="CI finished.",
        cwd=None,
        mode="forever",
        timeout_seconds=600,
        lifetime_timeout_seconds=3600,
        retry_exit_codes=[75],
        retry_delay_seconds=45,
        post_to="channel",
        deliver_key=None,
    )

    reloaded = ManagedWatchStore()
    saved = reloaded.get_watch(watch.id)
    sqlite = SQLiteBackgroundTaskStore(tmp_path / "state" / "vibe.sqlite")

    assert not (tmp_path / "state" / "watches.json").exists()
    assert saved is not None
    assert saved.session_id == "sesk8m4q2p7x"
    assert sqlite.get_watch(watch.id)["command"] == ["python3", "wait.py"]


def test_hfr_479_late_cycle_cannot_replace_committed_watch_terminal_outcome(
    tmp_path: Path,
) -> None:
    """HFR-479 -- the retiring cycle owns definition terminal fields forever."""

    sqlite = SQLiteBackgroundTaskStore(tmp_path / "state" / "vibe.sqlite")
    winner = ManagedWatchStore(tmp_path / "winner.json")
    winner._sqlite = sqlite
    late = ManagedWatchStore(tmp_path / "late.json")
    late._sqlite = sqlite
    watch = winner.add_watch(
        name="ordered retirement",
        session_key="",
        command=[],
        shell_command="true",
        prefix=None,
        cwd=None,
        mode="forever",
        timeout_seconds=0,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )
    late.load()
    late.maybe_reload()  # establish the invalidation baseline before retirement
    requests = TaskExecutionStore(tmp_path / "requests")
    requests._sqlite = sqlite

    assert winner.mark_cycle_result(
        watch.id,
        exit_code=7,
        error="terminal cycle failed",
        disable=True,
    )
    late_run = requests.build_hook_send(
        session_key="",
        prompt="late cycle evidence",
        run_type="watch",
        definition_id=watch.id,
        source_kind="watch",
    )
    assert late.mark_cycle_result(
        watch.id,
        exit_code=0,
        error=None,
        event_detected=True,
        disable=False,
        queued_run=late_run.to_dict(),
    )

    stored = sqlite.get_watch(watch.id)
    assert stored is not None
    assert stored["retired_at"] is not None
    assert stored["last_finished_at"] == stored["retired_at"]
    assert stored["last_exit_code"] == 7
    assert stored["last_error"] == "terminal cycle failed"
    assert sqlite.get_run(late_run.id) is not None


def test_cycle_start_preserves_last_completed_outcome_pair(tmp_path: Path) -> None:
    """Starting a cycle must not tear the last completed cycle's outcome pair."""

    store = ManagedWatchStore(tmp_path / "watches.json")
    watch = store.add_watch(
        name="retrying watch",
        session_key="",
        command=[],
        shell_command="exit 75",
        prefix=None,
        cwd=None,
        mode="forever",
        timeout_seconds=0,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )

    assert store.mark_cycle_result(
        watch.id,
        exit_code=75,
        error="watch command exited with status 75",
    )
    assert store.mark_cycle_start(watch.id)

    started = store.get_watch(watch.id)
    assert started is not None and started.last_started_at is not None
    assert (started.last_exit_code, started.last_error) == (
        75,
        "watch command exited with status 75",
    )


def test_sqlite_remove_watch_soft_deletes_watch_but_keeps_runtime(tmp_path: Path) -> None:
    sqlite = SQLiteBackgroundTaskStore(tmp_path / "state" / "vibe.sqlite")
    store = ManagedWatchStore(tmp_path / "watches.json")
    store._sqlite = sqlite
    watch = store.add_watch(
        name="Watch CI",
        session_key="slack::channel::C123",
        session_id="sesk8m4q2p7x",
        command=["python3", "wait.py"],
        shell_command=None,
        prefix="CI finished.",
        cwd=None,
        mode="forever",
        timeout_seconds=600,
        lifetime_timeout_seconds=3600,
        retry_exit_codes=[75],
        retry_delay_seconds=45,
        post_to="channel",
        deliver_key=None,
    )
    sqlite.write_watch_runtime(
        {
            "watches": {
                watch.id: {
                    "running": True,
                    "pid": _WORKER_PID,
                    "started_at": "2026-05-15T00:00:00+00:00",
                    "updated_at": "2026-05-15T00:00:00+00:00",
                }
            }
        },
        updated_at="2026-05-15T00:00:00+00:00",
    )

    assert store.remove_watch(watch.id) is True

    reloaded = ManagedWatchStore(tmp_path / "watches-reloaded.json")
    reloaded._sqlite = sqlite
    reloaded.load()

    assert reloaded.get_watch(watch.id) is None
    assert sqlite.get_watch(watch.id) is None
    assert sqlite.get_run(f"runtime:{watch.id}")["task_id"] == watch.id


def test_watch_runtime_store_uses_sqlite_when_path_is_default(tmp_path: Path, monkeypatch) -> None:
    monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
    store = WatchRuntimeStateStore()

    store.write(
        {
            "watches": {
                "watch-1": {
                    "running": True,
                    "pid": _WORKER_PID,
                    "started_at": "2026-05-15T00:00:00+00:00",
                    "updated_at": "2026-05-15T00:00:01+00:00",
                }
            }
        }
    )

    assert not (tmp_path / "runtime" / "watch_runtime.json").exists()
    assert store.load()["watches"]["watch-1"]["pid"] == _WORKER_PID


def test_managed_watch_service_once_success_enqueues_hook_and_disables(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Wait once",
        session_key="slack::channel::C123",
        command=["python3", "-c", "print('waiter output')"],
        shell_command=None,
        prefix="The waiter finished.",
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        for _ in range(100):
            if watch.id not in service._active_tasks:
                break
            await asyncio.sleep(0.05)
        await service.stop()

    asyncio.run(_run())

    pending = request_store.list_pending()
    saved = store.get_watch(watch.id)

    assert len(pending) == 1
    # ManagedWatchService enqueues with the dedicated "watch" run_type (core/watches.py),
    # which scheduled_tasks dispatches like a hook_send but tags as trigger_kind="watch".
    assert pending[0].request_type == "watch"
    assert pending[0].prompt == "The waiter finished.\n\nwaiter output"
    assert (
        pending[0].metadata[WATCH_HOOK_OUTCOME_METADATA_KEY]
        == WATCH_HOOK_OUTCOME_EVENT
    )
    assert saved is not None
    assert saved.enabled is False
    assert saved.last_exit_code == 0
    assert saved.last_event_at is not None


def test_managed_watch_service_does_not_enqueue_after_service_lease_loss(tmp_path: Path, monkeypatch) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Wait once",
        session_key="slack::channel::C123",
        command=["python3", "-c", "print('waiter output')"],
        shell_command=None,
        prefix="The waiter finished.",
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    service._running = True
    service._requires_service_lease = True
    owner_checks = iter([True, False])
    monkeypatch.setattr(
        "core.watches.runtime.current_process_owns_service_instance",
        lambda: next(owner_checks),
    )

    async def fake_run_cycle(*args, **kwargs):
        return _CycleResult(exit_code=0, stdout="waiter output", stderr="", timed_out=False)

    monkeypatch.setattr(service, "_run_cycle", fake_run_cycle)

    asyncio.run(service._run_watch(watch.id))

    assert request_store.list_pending() == []


def test_managed_watch_service_forever_timeout_disables_and_enqueues_failure(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Wait forever",
        session_key="slack::channel::C123",
        command=["python3", "-c", "import time; time.sleep(0.2)"],
        shell_command=None,
        prefix="Should stay silent.",
        cwd=None,
        mode="forever",
        timeout_seconds=0.05,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        watch_task = service._active_tasks[watch.id]
        try:
            await asyncio.wait_for(asyncio.shield(watch_task), timeout=10)
        finally:
            await service.stop()

    asyncio.run(_run())

    saved = store.get_watch(watch.id)

    pending = request_store.list_pending()
    assert saved is not None
    assert len(pending) == 1
    assert "stopped because the waiter timed out" in pending[0].prompt
    assert "Check whether the timeout is too short or the waiter is blocked" in pending[0].prompt
    assert saved.enabled is False
    assert saved.last_exit_code == 124


def test_managed_watch_service_forever_timeout_retries_when_explicitly_allowed(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Retry timeout forever",
        session_key="slack::channel::C123",
        command=["python3", "-c", "import time; time.sleep(0.2)"],
        shell_command=None,
        prefix="Should keep waiting.",
        cwd=None,
        mode="forever",
        timeout_seconds=0.05,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75, 124],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    cycles = 0
    watch_runs = 0
    run_cycle = service._run_cycle
    run_watch = service._run_watch

    async def counting_run_cycle(*args, **kwargs):
        nonlocal cycles
        cycles += 1
        return await run_cycle(*args, **kwargs)

    async def counting_run_watch(*args, **kwargs):
        nonlocal watch_runs
        watch_runs += 1
        return await run_watch(*args, **kwargs)

    monkeypatch.setattr(service, "_run_cycle", counting_run_cycle)
    monkeypatch.setattr(service, "_run_watch", counting_run_watch)

    async def _run() -> None:
        await _start_watch_service(service)
        # A fixed sleep bets that a real interpreter spawn, its 50ms deadline and
        # the write-back all land inside one window; on a contended runner they do
        # not, and the assertion samples ``last_exit_code`` while it is still None.
        #
        # Wait for the retry this test is named for instead, and name it as the
        # invariant that separates one from a restart: a *second waiter inside the
        # same watch run*. A watch that recorded the retryable timeout and then
        # returned would be started again by reconciliation and reach two waiters
        # too, so the second waiter alone does not distinguish the two.
        try:
            for _ in range(500):
                if cycles >= 2 and watch_runs == 1:
                    break
                await asyncio.sleep(0.02)
            else:
                raise AssertionError(
                    f"waiter never retried inside one watch run (cycles={cycles}, runs={watch_runs})"
                )
        finally:
            await service.stop()

    asyncio.run(_run())

    saved = store.get_watch(watch.id)
    assert saved is not None
    assert watch_runs == 1
    assert saved.enabled is True
    assert saved.last_exit_code == 124
    assert request_store.list_pending() == []


def test_managed_watch_service_stop_terminates_running_waiter(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Wait forever",
        session_key="slack::channel::C123",
        command=["python3", "-c", "import time; time.sleep(30)"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="forever",
        timeout_seconds=0,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> tuple[int, int | None]:
        await _start_watch_service(service)
        for _ in range(100):
            pid = service._active_pids.get(watch.id)
            if pid:
                break
            await asyncio.sleep(0.02)
        else:
            raise AssertionError("waiter pid was never recorded")
        pgid = os.getpgid(pid) if hasattr(os, "getpgid") else None
        await service.stop()
        return pid, pgid

    pid, pgid = asyncio.run(_run())

    if pgid is not None:
        assert pgid != os.getpgrp()

    with pytest.raises(ProcessLookupError):
        os.kill(pid, 0)


def test_managed_watch_service_lease_loss_terminates_running_waiter(tmp_path: Path, monkeypatch) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Wait forever",
        session_key="slack::channel::C123",
        command=["python3", "-c", "import time; time.sleep(30)"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="forever",
        timeout_seconds=0,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    service._requires_service_lease = True
    owner_state = {"owns": True}
    monkeypatch.setattr("core.watches.runtime.current_process_owns_service_instance", lambda: owner_state["owns"])

    async def _run() -> tuple[int, int | None]:
        await _start_watch_service(service)
        for _ in range(100):
            pid = service._active_pids.get(watch.id)
            if pid:
                break
            await asyncio.sleep(0.02)
        else:
            raise AssertionError("waiter pid was never recorded")
        pgid = os.getpgid(pid) if hasattr(os, "getpgid") else None
        active_tasks = list(service._active_tasks.values())
        owner_state["owns"] = False
        assert service._owns_service_instance() is False
        await asyncio.gather(*active_tasks, return_exceptions=True)
        if service._reconcile_task:
            try:
                await service._reconcile_task
            except asyncio.CancelledError:
                pass
            service._reconcile_task = None
        return pid, pgid

    pid, pgid = asyncio.run(_run())

    if pgid is not None:
        assert pgid != os.getpgrp()

    with pytest.raises(ProcessLookupError):
        os.kill(pid, 0)


def test_managed_watch_service_records_non_secret_process_identity(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    secret = "runtime-expanded-secret"
    monkeypatch.setenv("WATCH_RUNTIME_SECRET", secret)
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Wait forever",
        session_key="slack::channel::C123",
        command=[],
        shell_command=(
            "exec python3 -c 'import time; time.sleep(30)' "
            '"$WATCH_RUNTIME_SECRET"'
        ),
        prefix=None,
        cwd=None,
        mode="forever",
        timeout_seconds=0,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> dict[str, object]:
        await _start_watch_service(service)
        for _ in range(100):
            entry = runtime_store.load().get("watches", {}).get(watch.id, {})
            if entry.get("process_identity"):
                await service.stop()
                return entry
            await asyncio.sleep(0.02)
        await service.stop()
        raise AssertionError("process identity was never written")

    entry = asyncio.run(_run())
    identity = entry["process_identity"]
    assert isinstance(entry["pid"], int)
    assert datetime.fromisoformat(str(entry["started_at"])).year >= 2024
    assert isinstance(identity, dict)
    assert identity["pid"] == entry["pid"]
    assert identity["create_time"] > 0
    assert identity["worker_fingerprint"].startswith("sha256:")
    assert "cmdline" not in identity
    assert secret not in json.dumps(entry)


def test_managed_watch_service_turns_spawn_error_into_failed_cycle(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Broken waiter",
        session_key="slack::channel::C123",
        command=["/definitely/missing/waiter"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        for _ in range(100):
            if watch.id not in service._active_tasks:
                break
            await asyncio.sleep(0.02)
        await service.stop()

    asyncio.run(_run())

    saved = store.get_watch(watch.id)
    pending = request_store.list_pending()
    assert saved is not None
    assert saved.enabled is False
    assert saved.last_exit_code == 1
    assert saved.last_error
    assert len(pending) == 1
    assert "stopped because the waiter exited with code 1" in pending[0].prompt
    assert "fix the waiter or its dependencies" in pending[0].prompt
    assert (
        pending[0].metadata[WATCH_HOOK_OUTCOME_METADATA_KEY]
        == WATCH_HOOK_OUTCOME_WAITER_FAILURE
    )


@pytest.mark.parametrize("use_shell", [False, True], ids=["exec", "shell"])
def test_managed_watch_service_stops_and_notifies_when_cwd_was_removed(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    use_shell: bool,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    removed_cwd = tmp_path / "removed-worktree"
    removed_cwd.mkdir()
    removed_cwd.rmdir()
    watch = store.add_watch(
        name="PR review",
        session_key="",
        session_id="ses-linked",
        command=[] if use_shell else [sys.executable, "-c", "raise AssertionError('must not run')"],
        shell_command="exit 99" if use_shell else None,
        prefix=None,
        message="PR review activity detected.",
        cwd=str(removed_cwd),
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    run_cycle_called = False

    async def unexpected_run_cycle(*_args, **_kwargs):
        nonlocal run_cycle_called
        run_cycle_called = True
        raise AssertionError("waiter must not spawn when its cwd is missing")

    monkeypatch.setattr(service, "_run_cycle", unexpected_run_cycle)
    service._running = True
    asyncio.run(service._run_watch(watch.id))

    saved = store.get_watch(watch.id)
    pending = request_store.list_pending()
    assert run_cycle_called is False
    assert saved is not None
    assert saved.enabled is False
    assert saved.last_exit_code == 1
    assert saved.last_error == (f"watch working directory no longer exists or is not a directory: {removed_cwd}")
    assert len(pending) == 1
    assert pending[0].session_id == "ses-linked"
    assert pending[0].task_id == watch.id
    assert pending[0].source_kind == "watch"
    assert pending[0].prompt == (
        "Watch 'PR review' stopped because its working directory is no longer available.\n"
        f"Working directory: {removed_cwd}\n"
        "Update or recreate the watch with a valid cwd before monitoring continues."
    )


def test_missing_watch_cwd_does_not_stop_other_watches(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    removed_cwd = tmp_path / "removed-worktree"
    broken = store.add_watch(
        name="Broken watch",
        session_key="slack::channel::C123",
        command=[sys.executable, "-c", "raise AssertionError('must not run')"],
        shell_command=None,
        prefix=None,
        cwd=str(removed_cwd),
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )
    healthy = store.add_watch(
        name="Healthy watch",
        session_key="slack::channel::C123",
        command=[sys.executable, "-c", "print('healthy completed')"],
        shell_command=None,
        prefix="Healthy watch event.",
        cwd=str(tmp_path),
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=30,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        for _ in range(100):
            broken_saved = store.get_watch(broken.id)
            healthy_saved = store.get_watch(healthy.id)
            if broken_saved and healthy_saved and not broken_saved.enabled and not healthy_saved.enabled:
                break
            await asyncio.sleep(0.02)
        await service.stop()

    asyncio.run(_run())

    broken_saved = store.get_watch(broken.id)
    healthy_saved = store.get_watch(healthy.id)
    prompts = [request.prompt for request in request_store.list_pending()]
    assert broken_saved is not None
    assert broken_saved.last_error and str(removed_cwd) in broken_saved.last_error
    assert healthy_saved is not None
    assert healthy_saved.last_error is None
    assert healthy_saved.last_event_at is not None
    assert any("working directory is no longer available" in (prompt or "") for prompt in prompts)
    assert any("healthy completed" in (prompt or "") for prompt in prompts)


def test_managed_watch_service_forever_retries_only_allowed_exit_code(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Retry waiter",
        session_key="slack::channel::C123",
        command=[sys.executable, "-c", "import sys; sys.exit(75)"],
        shell_command=None,
        prefix="Retry only.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        deadline = asyncio.get_running_loop().time() + 2
        while asyncio.get_running_loop().time() < deadline:
            saved = store.get_watch(watch.id)
            if saved is not None and saved.last_exit_code == 75:
                break
            await asyncio.sleep(0.01)
        await service.stop()

    asyncio.run(_run())

    saved = store.get_watch(watch.id)
    assert saved is not None
    assert saved.enabled is True
    assert saved.last_exit_code == 75
    assert request_store.list_pending() == []


def test_managed_watch_service_once_retries_until_first_success(
    tmp_path: Path,
    monkeypatch,
) -> None:
    """HFR-463: a retry result keeps a once Watch armed until its first event."""

    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Wait until ready",
        session_key="slack::channel::C123",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix="Continue after the event.",
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    results = iter(
        [
            _CycleResult(exit_code=75, stdout="", stderr="not ready", timed_out=False),
            _CycleResult(exit_code=0, stdout="ready", stderr="", timed_out=False),
        ]
    )
    calls = 0

    async def fake_run_cycle(*args, **kwargs):
        nonlocal calls
        calls += 1
        return next(results)

    monkeypatch.setattr(service, "_run_cycle", fake_run_cycle)
    service._running = True
    service._requires_service_lease = False

    asyncio.run(service._run_watch(watch.id))

    saved = store.get_watch(watch.id)
    pending = request_store.list_pending()
    assert calls == 2
    assert saved is not None and saved.enabled is False
    assert saved.last_exit_code == 0
    assert len(pending) == 1
    assert pending[0].prompt.endswith("ready")


@pytest.mark.parametrize(
    ("language", "expected_prefix", "expected_body"),
    [
        (
            "en",
            "Watch stopped after reaching its lifetime timeout.",
            "Watch 'Bounded wait' reached its lifetime timeout after 0 second(s).",
        ),
        (
            "zh",
            "Watch 达到生命周期上限后已停止。",
            "Watch「Bounded wait」已达到 0 秒的生命周期上限。",
        ),
    ],
)
def test_retrying_once_watch_lifetime_bounds_a_long_retry_delay(
    tmp_path: Path,
    monkeypatch,
    language: str,
    expected_prefix: str,
    expected_body: str,
) -> None:
    """HFR-470: the overall lifetime truncates a once Watch retry delay."""

    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    watch = store.add_watch(
        name="Bounded wait",
        session_key="slack::channel::C123",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0.05,
        retry_exit_codes=[75],
        retry_delay_seconds=10,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(config=SimpleNamespace(language=language)),
        store=store,
        request_store=request_store,
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )

    async def fake_run_cycle(*args, **kwargs):
        return _CycleResult(exit_code=75, stdout="", stderr="not ready", timed_out=False)

    monkeypatch.setattr(service, "_run_cycle", fake_run_cycle)
    service._running = True
    service._requires_service_lease = False

    started = time.monotonic()
    asyncio.run(service._run_watch(watch.id))

    saved = store.get_watch(watch.id)
    assert time.monotonic() - started < 1
    assert saved is not None and saved.enabled is False
    assert saved.last_exit_code == 124
    pending = request_store.list_pending()
    assert len(pending) == 1
    assert expected_prefix in pending[0].prompt
    assert expected_body in pending[0].prompt


@pytest.mark.parametrize("mode", ["once", "forever"])
@pytest.mark.parametrize("origin_source", ["metadata", "legacy_created_at"])
def test_watch_lifetime_origin_survives_supervisor_restart(
    tmp_path: Path,
    monkeypatch,
    mode: str,
    origin_source: str,
) -> None:
    """HFR-470: one armed episode keeps its deadline across supervisor restarts."""

    path = tmp_path / "watches.json"
    store = ManagedWatchStore(path)
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    watch = store.add_watch(
        name="Bounded wait",
        session_key="slack::channel::C123",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode=mode,
        timeout_seconds=5,
        lifetime_timeout_seconds=1,
        retry_exit_codes=[75],
        retry_delay_seconds=10,
        post_to=None,
        deliver_key=None,
    )
    expired_origin = (datetime.now(timezone.utc) - timedelta(seconds=5)).isoformat()
    if origin_source == "metadata":
        watch.metadata[LIFETIME_STARTED_AT_METADATA_KEY] = expired_origin
    else:
        watch.metadata.pop(LIFETIME_STARTED_AT_METADATA_KEY)
        watch.created_at = expired_origin
    store.upsert_watch(watch)

    restarted_store = ManagedWatchStore(path)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=restarted_store,
        request_store=request_store,
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )

    async def unexpected_cycle(*args, **kwargs):
        raise AssertionError("a supervisor restart must not grant another cycle")

    monkeypatch.setattr(service, "_run_cycle", unexpected_cycle)
    service._running = True
    service._requires_service_lease = False

    asyncio.run(service._run_watch(watch.id))

    saved = restarted_store.get_watch(watch.id)
    assert saved is not None and saved.enabled is False
    assert saved.last_exit_code == 124
    assert len(request_store.list_pending()) == 1


@pytest.mark.parametrize("metadata_run_id", [None, "stale-run"])
def test_watch_lifetime_retires_while_an_existing_follow_up_is_active(
    tmp_path: Path,
    monkeypatch,
    metadata_run_id: str | None,
) -> None:
    """The actual queued Run owns expiry even when Watch metadata is absent or stale."""

    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    watch = store.add_watch(
        name="Bounded follow-up",
        session_key="slack::channel::C123",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix="Inspect the event.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0.05,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )
    if metadata_run_id is not None:
        assert store.mark_cycle_result(
            watch.id,
            exit_code=None,
            error=None,
            metadata_updates={FOLLOW_UP_RUN_ID_METADATA_KEY: metadata_run_id},
        )
    active = request_store.enqueue_hook_send(
        session_key=watch.session_key,
        prompt="existing event",
        run_type="watch",
        definition_id=watch.id,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )

    async def unexpected_cycle(*args, **kwargs):
        raise AssertionError("the follow-up fence must not admit another waiter")

    monkeypatch.setattr(service, "_run_cycle", unexpected_cycle)
    monkeypatch.setattr(watches_module, "WATCH_FOLLOW_UP_POLL_SECONDS", 0.005)
    service._running = True
    service._requires_service_lease = False

    started = time.monotonic()
    asyncio.run(service._run_watch(watch.id))

    saved = store.get_watch(watch.id)
    assert time.monotonic() - started < 1
    assert saved is not None and saved.enabled is False
    assert saved.last_exit_code == 124
    assert "retired without starting another Run" in str(saved.last_error)
    assert active.id in str(saved.last_error)
    pending = request_store.list_pending()
    assert [request.id for request in pending] == [active.id]


def test_once_watch_retries_timeout_only_when_124_is_allowed(
    tmp_path: Path,
    monkeypatch,
) -> None:
    """HFR-471: timeout retries require explicit policy on a once Watch."""

    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    watch = store.add_watch(
        name="Retry one timeout",
        session_key="slack::channel::C123",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix="Continue after the event.",
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[124],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    results = iter(
        [
            _CycleResult(exit_code=124, stdout="", stderr="", timed_out=True),
            _CycleResult(exit_code=0, stdout="ready", stderr="", timed_out=False),
        ]
    )

    async def fake_run_cycle(*args, **kwargs):
        return next(results)

    monkeypatch.setattr(service, "_run_cycle", fake_run_cycle)
    service._running = True
    service._requires_service_lease = False

    asyncio.run(service._run_watch(watch.id))

    saved = store.get_watch(watch.id)
    assert saved is not None and saved.enabled is False
    assert saved.last_exit_code == 0
    assert len(request_store.list_pending()) == 1


def test_forever_watch_waits_for_previous_follow_up_before_rearming(
    tmp_path: Path,
    monkeypatch,
) -> None:
    """HFR-464: queued/running follow-ups and cooldown both hold re-arm closed."""

    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Disk pressure",
        session_key="slack::channel::C123",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix="Inspect the new pressure event.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    calls = 0

    async def fake_run_cycle(*args, **kwargs):
        nonlocal calls
        calls += 1
        if calls == 1:
            return _CycleResult(exit_code=0, stdout="event 1", stderr="", timed_out=False)
        service._running = False
        return _CycleResult(
            exit_code=NO_EVENT_EXIT_CODE,
            stdout="",
            stderr=NO_EVENT_MARKER,
            timed_out=False,
        )

    monkeypatch.setattr(service, "_run_cycle", fake_run_cycle)
    monkeypatch.setattr(watches_module, "WATCH_MIN_REARM_SECONDS", 0.1)
    # The production fence uses wall time for the cooldown and polls for both
    # unsettled Runs and the cooldown boundary. Drive that clock explicitly and
    # use zero-delay yields so the test never infers elapsed time from a loaded
    # runner's sleep duration.
    clock = _ControlledWatchClock(datetime.now(timezone.utc))
    monkeypatch.setattr(watches_module, "datetime", clock)
    monkeypatch.setattr(watches_module, "WATCH_FOLLOW_UP_POLL_SECONDS", 0)
    service._running = True
    service._requires_service_lease = False

    async def _run() -> None:
        task = asyncio.create_task(service._run_watch(watch.id))
        async def wait_for_pending() -> None:
            while not request_store.list_pending():
                await asyncio.sleep(0)

        await asyncio.wait_for(wait_for_pending(), timeout=1)
        assert calls == 1
        for _ in range(8):
            await asyncio.sleep(0)
            assert calls == 1, "an unfinished event Run must hold the waiter closed"
        request = request_store.list_pending()[0]
        claimed = request_store.claim(request.id)
        assert claimed is not None
        request_store.complete(claimed, ok=True)
        settled = request_store.get_run(request.id)
        assert settled is not None
        assert settled["status"] == "succeeded"
        completed_at = datetime.fromisoformat(settled["completed_at"])
        clock.current = completed_at + timedelta(seconds=0.099)
        for _ in range(8):
            await asyncio.sleep(0)
            assert calls == 1, "the cooldown must hold the waiter closed"
        clock.current = completed_at + timedelta(seconds=0.101)
        await asyncio.wait_for(task, timeout=2)

    asyncio.run(_run())

    assert calls == 2


def test_atomic_watch_outbox_rejects_a_second_unsettled_follow_up(
    tmp_path: Path,
) -> None:
    """HFR-466: atomic admission rejects a second unfinished Watch follow-up."""

    store = ManagedWatchStore()
    request_store = TaskExecutionStore()
    assert store.sqlite_backend is not None
    assert request_store.sqlite_backend is not None
    assert store.sqlite_backend.db_path == request_store.sqlite_backend.db_path
    session_id = _bare_watch_session_row(workdir=tmp_path, anchor="single_flight")
    watch = store.add_watch(
        name="Single-flight watch",
        session_key="",
        session_id=session_id,
        session_policy="existing",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix="Inspect it.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )

    assert service._commit_cycle_result(
        watch,
        exit_code=0,
        error=None,
        event_detected=True,
        prompt="first event",
    )
    reloaded = store.get_watch(watch.id)
    assert reloaded is not None
    assert not service._commit_cycle_result(
        reloaded,
        exit_code=0,
        error=None,
        event_detected=True,
        prompt="duplicate event",
    )

    assert len(request_store.list_pending()) == 1
    unsettled = request_store.get_unsettled_watch_run(watch.id)
    assert unsettled is not None
    assert unsettled["prompt"] == "first event"
    saved = store.get_watch(watch.id)
    assert saved is not None
    assert saved.metadata[DELIVERY_ACK_METADATA_KEY] == 1


def test_forever_watch_sixth_rapid_success_pauses_and_queues_one_repair_run(
    tmp_path: Path,
    monkeypatch,
) -> None:
    """HFR-465: the sixth rapid event pauses once and preserves repair evidence."""

    store = ManagedWatchStore()
    request_store = TaskExecutionStore()
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    session_id = _bare_watch_session_row(workdir=tmp_path, anchor="circuit_repair")
    watch = store.add_watch(
        name="Level-triggered disk alert",
        session_key="",
        session_id=session_id,
        session_policy="existing",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix="Inspect the disk alert.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    calls = 0

    async def fake_run_cycle(*args, **kwargs):
        nonlocal calls
        calls += 1
        stdout = f"persistent level {calls}"
        stderr = "diagnostic stderr"
        if calls == 6:
            stdout = ("old stdout\n" * 500) + stdout
            stderr = ("old stderr\n" * 500) + stderr
        return _CycleResult(
            exit_code=0,
            stdout=stdout,
            stderr=stderr,
            timed_out=False,
        )

    monkeypatch.setattr(service, "_run_cycle", fake_run_cycle)
    monkeypatch.setattr(watches_module, "WATCH_MIN_REARM_SECONDS", 0)
    monkeypatch.setattr(watches_module, "WATCH_FOLLOW_UP_POLL_SECONDS", 0.001)
    service._running = True
    service._requires_service_lease = False

    async def _run() -> None:
        task = asyncio.create_task(service._run_watch(watch.id))
        while not task.done():
            for request in request_store.list_pending():
                outcome = request.metadata.get(WATCH_HOOK_OUTCOME_METADATA_KEY)
                if outcome != WATCH_HOOK_OUTCOME_CIRCUIT_REPAIR:
                    claimed = request_store.claim(request.id)
                    assert claimed is not None
                    request_store.complete(claimed, ok=True)
            await asyncio.sleep(0.001)
        await task

    asyncio.run(_run())

    saved = store.get_watch(watch.id)
    pending = request_store.list_pending()
    completed = request_store.list_runs(status="succeeded")
    assert calls == 6
    assert saved is not None and saved.enabled is False
    assert saved.retired_at is None
    assert saved.last_error is not None
    assert "60 seconds" in saved.last_error
    assert saved.metadata[DELIVERY_ACK_METADATA_KEY] == 5
    assert len(saved.metadata[RECENT_EVENT_TIMESTAMPS_METADATA_KEY]) == 6
    incident = saved.metadata[CIRCUIT_BREAKER_METADATA_KEY]
    assert incident["repair_run_id"] == saved.metadata[FOLLOW_UP_RUN_ID_METADATA_KEY]
    assert len(incident["stdout"]) <= WATCH_CIRCUIT_OUTPUT_LIMIT
    assert len(incident["stderr"]) <= WATCH_CIRCUIT_OUTPUT_LIMIT
    assert incident["stdout"].endswith("persistent level 6")
    assert incident["stderr"].endswith("diagnostic stderr")
    assert len(completed) == 5
    assert len(pending) == 1
    assert pending[0].id in saved.last_error
    assert (
        pending[0].metadata[WATCH_HOOK_OUTCOME_METADATA_KEY]
        == WATCH_HOOK_OUTCOME_CIRCUIT_REPAIR
    )
    assert "Level-triggered disk alert" in pending[0].prompt
    assert "persistent level 6" in pending[0].prompt
    assert "allowed retry exit codes: 75" in pending[0].prompt
    assert "vibe watch resume" in pending[0].prompt

    restarted = ManagedWatchService(
        controller=SimpleNamespace(),
        store=ManagedWatchStore(),
        request_store=request_store,
        runtime_store=runtime_store,
    )
    restarted._running = True
    restarted._requires_service_lease = False
    asyncio.run(restarted._run_watch(watch.id))
    assert len(request_store.list_pending()) == 1


def test_watch_event_burst_window_excludes_successes_older_than_sixty_seconds() -> None:
    now = datetime(2026, 8, 10, 12, 0, tzinfo=timezone.utc)
    timestamps = [
        (now - timedelta(seconds=61)).isoformat(),
        (now - timedelta(seconds=59)).isoformat(),
        now.isoformat(),
    ]

    recent = _recent_event_timestamps(
        {RECENT_EVENT_TIMESTAMPS_METADATA_KEY: timestamps},
        now=now,
    )

    assert recent == timestamps[1:]


def test_resuming_circuit_paused_watch_clears_window_but_keeps_repair_fence(
    tmp_path: Path,
) -> None:
    """HFR-467: resume clears burst state without releasing the active repair Run."""

    store = ManagedWatchStore(tmp_path / "watches.json")
    watch = store.add_watch(
        name="Disk alert",
        session_key="slack::channel::C123",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
        metadata={
            FOLLOW_UP_RUN_ID_METADATA_KEY: "run-repair",
            RECENT_EVENT_TIMESTAMPS_METADATA_KEY: ["2026-08-10T00:00:00+00:00"],
            CIRCUIT_BREAKER_METADATA_KEY: {
                "status": "tripped",
                "repair_run_id": "run-repair",
            },
        },
    )
    store.set_enabled(watch.id, False)
    prior_lifetime_origin = watch.metadata[LIFETIME_STARTED_AT_METADATA_KEY]

    resumed = store.set_enabled(watch.id, True)

    assert RECENT_EVENT_TIMESTAMPS_METADATA_KEY not in resumed.metadata
    assert resumed.metadata[FOLLOW_UP_RUN_ID_METADATA_KEY] == "run-repair"
    assert resumed.metadata[CIRCUIT_BREAKER_METADATA_KEY]["status"] == "resumed"
    assert resumed.metadata[LIFETIME_STARTED_AT_METADATA_KEY] != prior_lifetime_origin


def test_storage_resume_applies_the_same_circuit_reset_as_cli() -> None:
    """HFR-468: storage and CLI resume paths apply the same circuit reset."""

    store = ManagedWatchStore()
    assert store.sqlite_backend is not None
    watch = store.add_watch(
        name="Disk alert",
        session_key="slack::channel::C123",
        command=[sys.executable, "wait.py"],
        shell_command=None,
        prefix=None,
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
        metadata={
            FOLLOW_UP_RUN_ID_METADATA_KEY: "run-repair",
            RECENT_EVENT_TIMESTAMPS_METADATA_KEY: ["2026-08-10T00:00:00+00:00"],
            CIRCUIT_BREAKER_METADATA_KEY: {
                "status": "tripped",
                "repair_run_id": "run-repair",
            },
        },
    )
    assert store.sqlite_backend.set_definition_enabled(
        watch.id,
        False,
        definition_type="watch",
    )
    prior_lifetime_origin = watch.metadata[LIFETIME_STARTED_AT_METADATA_KEY]
    assert store.sqlite_backend.set_definition_enabled(
        watch.id,
        True,
        definition_type="watch",
    )

    store.load()
    resumed = store.get_watch(watch.id)
    assert resumed is not None
    assert RECENT_EVENT_TIMESTAMPS_METADATA_KEY not in resumed.metadata
    assert resumed.metadata[FOLLOW_UP_RUN_ID_METADATA_KEY] == "run-repair"
    assert resumed.metadata[CIRCUIT_BREAKER_METADATA_KEY]["status"] == "resumed"
    assert resumed.metadata[LIFETIME_STARTED_AT_METADATA_KEY] != prior_lifetime_origin


def test_changing_watch_cwd_starts_a_fresh_burst_window(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    old_cwd = tmp_path / "old"
    new_cwd = tmp_path / "new"
    watch = store.add_watch(
        name="Relative waiter",
        session_key="slack::channel::C123",
        command=["./wait.py"],
        shell_command=None,
        prefix="Inspect it.",
        cwd=str(old_cwd),
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
        metadata={
            RECENT_EVENT_TIMESTAMPS_METADATA_KEY: ["2026-08-10T00:00:00+00:00"],
            FOLLOW_UP_RUN_ID_METADATA_KEY: "run-before-cwd-change",
        },
    )
    lifetime_origin = watch.metadata[LIFETIME_STARTED_AT_METADATA_KEY]

    updated = store.update_watch(
        watch.id,
        name=watch.name,
        session_key=watch.session_key,
        session_id=watch.session_id,
        command=watch.command,
        shell_command=watch.shell_command,
        prefix=watch.prefix,
        cwd=str(new_cwd),
        mode=watch.mode,
        timeout_seconds=watch.timeout_seconds,
        lifetime_timeout_seconds=watch.lifetime_timeout_seconds,
        retry_exit_codes=watch.retry_exit_codes,
        retry_delay_seconds=watch.retry_delay_seconds,
        post_to=watch.post_to,
        deliver_key=watch.deliver_key,
        agent_name=watch.agent_name,
        session_policy=watch.session_policy,
        message=watch.message,
        metadata=watch.metadata,
    )

    assert RECENT_EVENT_TIMESTAMPS_METADATA_KEY not in updated.metadata
    assert updated.metadata[FOLLOW_UP_RUN_ID_METADATA_KEY] == "run-before-cwd-change"
    assert updated.metadata[LIFETIME_STARTED_AT_METADATA_KEY] == lifetime_origin


def test_managed_watch_service_once_no_event_exit_finishes_without_follow_up(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Green CI waiter",
        session_key="slack::channel::C123",
        command=_quiet_waiter_command(),
        shell_command=None,
        prefix="CI failed. Fix it.",
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        service.start()
        for _ in range(100):
            saved = store.get_watch(watch.id)
            if saved and not saved.enabled:
                break
            await asyncio.sleep(0.02)
        await service.stop()

    asyncio.run(_run())

    saved = store.get_watch(watch.id)
    assert saved is not None
    assert saved.enabled is False
    assert saved.last_exit_code == NO_EVENT_EXIT_CODE
    assert saved.last_error is None
    # The whole point: a completed cycle that found nothing costs no Agent turn.
    assert request_store.list_pending() == []


def test_managed_watch_service_logs_what_a_quiet_cycle_suppressed(tmp_path: Path, caplog) -> None:
    """A cycle that reports nothing still has to leave its summary somewhere.

    No hook carries a no-event cycle's output, so the waiter's own account of what
    it saw and chose not to report — the green CI table, the filtered comments —
    died with the process. ``--only-on-failure`` documents that summary as
    inspectable, which it was not.
    """
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Green CI waiter",
        session_key="slack::channel::C123",
        command=_quiet_waiter_command("All watched workflows succeeded: lint, build"),
        shell_command=None,
        prefix="CI failed. Fix it.",
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        service.start()
        for _ in range(100):
            saved = store.get_watch(watch.id)
            if saved and not saved.enabled:
                break
            await asyncio.sleep(0.02)
        await service.stop()

    with caplog.at_level(logging.INFO, logger="core.watches"):
        asyncio.run(_run())

    assert request_store.list_pending() == []
    quiet_logs = [record.getMessage() for record in caplog.records if "found nothing to report" in record.getMessage()]
    assert quiet_logs, caplog.text
    assert watch.id in quiet_logs[0]
    assert "All watched workflows succeeded: lint, build" in quiet_logs[0]


def test_managed_watch_service_forever_no_event_exit_rearms_without_follow_up(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Quiet forever waiter",
        session_key="slack::channel::C123",
        command=_quiet_waiter_command(),
        shell_command=None,
        prefix="Something happened.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        deadline = asyncio.get_running_loop().time() + 2
        while asyncio.get_running_loop().time() < deadline:
            saved = store.get_watch(watch.id)
            if saved is not None and saved.last_exit_code == NO_EVENT_EXIT_CODE:
                break
            await asyncio.sleep(0.01)
        await service.stop()

    asyncio.run(_run())

    saved = store.get_watch(watch.id)
    assert saved is not None
    assert saved.enabled is True
    assert saved.last_exit_code == NO_EVENT_EXIT_CODE
    assert saved.last_error is None
    assert request_store.list_pending() == []


def test_managed_watch_service_treats_an_unmarked_exit_64_as_a_failure(tmp_path: Path) -> None:
    """64 is ``sysexits`` EX_USAGE, not a private Avibe signal.

    A generic watched command that rejects its own arguments exits 64. Reading that
    as a quiet cycle would swallow the failure notice the user relies on and, in
    forever mode, rerun the broken command for as long as the watch lives.
    """
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Misconfigured forever waiter",
        session_key="slack::channel::C123",
        command=[
            sys.executable,
            "-c",
            f"import sys; print('usage: mytool [--flag]', file=sys.stderr); sys.exit({NO_EVENT_EXIT_CODE})",
        ],
        shell_command=None,
        prefix="Investigate the failure.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        for _ in range(100):
            if watch.id not in service._active_tasks:
                break
            await asyncio.sleep(0.02)
        await service.stop()

    asyncio.run(_run())

    saved = store.get_watch(watch.id)
    pending = request_store.list_pending()
    assert saved is not None
    # Stopped, recorded with its error text, and reported once — as before the
    # no-event code existed.
    assert saved.enabled is False
    assert saved.last_exit_code == NO_EVENT_EXIT_CODE
    assert saved.last_error and "usage: mytool" in saved.last_error
    assert len(pending) == 1
    assert "usage: mytool" in pending[0].prompt


def test_managed_watch_service_forever_non_retry_error_disables_and_enqueues_failure(tmp_path: Path) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Broken forever waiter",
        session_key="slack::channel::C123",
        command=["python3", "-c", "import sys; sys.exit(1)"],
        shell_command=None,
        prefix="Investigate the failure.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        for _ in range(100):
            if watch.id not in service._active_tasks:
                break
            await asyncio.sleep(0.02)
        await service.stop()

    asyncio.run(_run())

    saved = store.get_watch(watch.id)
    pending = request_store.list_pending()
    assert saved is not None
    assert saved.enabled is False
    assert saved.last_exit_code == 1
    assert saved.last_error
    assert len(pending) == 1
    assert pending[0].prompt.startswith("Investigate the failure.\n\nWatch 'Broken forever waiter' stopped because the waiter exited with code 1.")


def test_managed_watch_service_backs_off_one_watch_after_its_store_error(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A store error stops one watch for a doubling backoff, not the whole service.

    The watch sits out each backoff instead of restarting into the same error, and
    runs again once the store accepts its cycle result. A watch created meanwhile
    starts at once: a store error one watch saw says nothing about the others, and
    a service-wide fuse left every Watch created after a brief disk-full episode
    unstarted until the service restarted.
    """

    monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
    monkeypatch.setattr("core.watches.WATCH_STORE_RETRY_INITIAL_SECONDS", 0.2)

    class FlakyResultStore(ManagedWatchStore):
        def __init__(self, path: Path):
            super().__init__(path)
            self.starts: dict[str, list[float]] = {}
            self.failures: dict[str, list[float]] = {}
            self.failing: dict[str, int] = {}

        def mark_cycle_start(self, watch_id: str) -> bool:
            self.starts.setdefault(watch_id, []).append(time.monotonic())
            return super().mark_cycle_start(watch_id)

        def mark_cycle_result(self, watch_id: str, *args, **kwargs) -> bool:
            if self.failing.get(watch_id, 0) > 0:
                self.failing[watch_id] -= 1
                self.failures.setdefault(watch_id, []).append(time.monotonic())
                raise RuntimeError("unable to open database file")
            return super().mark_cycle_result(watch_id, *args, **kwargs)

    def add_retrying_watch(name: str) -> watches_module.ManagedWatch:
        return store.add_watch(
            name=name,
            session_key="slack::channel::C123",
            command=[sys.executable, "-c", "import sys; sys.exit(75)"],
            shell_command=None,
            prefix="Should not storm.",
            cwd=None,
            mode="forever",
            timeout_seconds=5,
            lifetime_timeout_seconds=0,
            retry_exit_codes=[75],
            retry_delay_seconds=0.01,
            post_to=None,
            deliver_key=None,
        )

    store = FlakyResultStore(tmp_path / "watches.json")
    broken = add_retrying_watch("Broken persistence")
    store.failing[broken.id] = 2
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )

    async def _run() -> None:
        await _start_watch_service(service)
        await _await_backed_off_watch(service, broken.id)
        created = add_retrying_watch("Created during the outage")
        await _await_condition(
            lambda: created.id in store.starts,
            "a watch created after another watch's store error never started",
        )
        await _await_condition(
            lambda: store.failing[broken.id] == 0 and broken.id not in service._watch_store_retries,
            "the backed-off watch never committed a cycle after the store recovered",
        )
        await service.stop()

    asyncio.run(_run())

    starts = store.starts[broken.id]
    failures = store.failures[broken.id]
    assert len(failures) == 2
    assert len(starts) >= 3
    assert starts[1] - failures[0] >= 0.2, "the watch restarted before its backoff elapsed"
    assert starts[2] - failures[1] >= 0.4, "the second backoff did not double"


def test_managed_watch_service_backs_off_quiet_cycle_after_store_error(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A quiet cycle commits through the same wrapper as every other result branch.

    Called synchronously, a raising ``mark_cycle_result`` escapes ``_run_cycle`` and
    kills the watch task with no backoff and the cycle unrecorded. Reconcile then
    restarts the same quiet cycle and repeats it, so a real storage failure never
    reaches the user as one.
    """

    monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)

    class FailingResultStore(ManagedWatchStore):
        def __init__(self, path: Path):
            super().__init__(path)
            self.starts = 0

        def mark_cycle_start(self, watch_id: str) -> bool:
            self.starts += 1
            return super().mark_cycle_start(watch_id)

        def mark_cycle_result(self, *args, **kwargs) -> bool:
            raise RuntimeError("database disk image is malformed")

    store = FailingResultStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = store.add_watch(
        name="Quiet with broken persistence",
        session_key="slack::channel::C123",
        command=_quiet_waiter_command("nothing to report"),
        shell_command=None,
        prefix="Should not storm.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        await _await_backed_off_watch(service, watch.id)
        # Settle across several reconcile ticks: the backoff must keep the quiet
        # cycle from starting again.
        await asyncio.sleep(0.08)
        await service.stop()

    asyncio.run(_run())

    assert store.starts == 1, "reconcile restarted the quiet cycle inside its backoff"
    # A quiet cycle authorises no hook, and a failed one must not invent it.
    assert request_store.list_pending() == []


def test_managed_watch_service_resumes_reconcile_after_store_outage(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    caplog: pytest.LogCaptureFixture,
) -> None:
    """The polling loop pauses on a store outage and resumes when the store heals.

    A pause quiets the logs; it must not stop the probing. Before the fix the
    loop stopped reading after three errors, so a watch the store listed again
    once the outage ended never started until the service restarted.
    """

    monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
    outage_reads = watches_module.WATCH_STORE_RECONCILE_PAUSE_FAILURES + 3

    class OutageListStore(ManagedWatchStore):
        def __init__(self, path: Path):
            super().__init__(path)
            self.failures_remaining = outage_reads

        def list_watches(self):
            if self.failures_remaining > 0:
                self.failures_remaining -= 1
                raise RuntimeError("unable to open database file")
            return super().list_watches()

    store = OutageListStore(tmp_path / "watches.json")
    watch = store.add_watch(
        name="Waiting out the outage",
        session_key="slack::channel::C123",
        command=[sys.executable, "-c", "import time; time.sleep(30)"],
        shell_command=None,
        prefix="Done.",
        cwd=None,
        mode="once",
        timeout_seconds=60,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[],
        retry_delay_seconds=0,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )

    async def _run() -> None:
        await _start_watch_service(service)
        await _await_condition(
            lambda: watch.id in service._active_tasks,
            "the watch never started after the store outage ended",
        )
        await service.stop()

    with caplog.at_level(logging.INFO, logger="core.watches"):
        asyncio.run(_run())

    assert store.failures_remaining == 0
    assert service._store_reconcile_failures == 0
    pauses = [record for record in caplog.records if "Pausing managed watch reconciliation" in record.message]
    assert len(pauses) == 1, "the pause must be reported once, not per probe"
    assert any("reconciliation resumed" in record.message for record in caplog.records)


def test_managed_watch_service_idle_tick_does_not_write_runtime_state(tmp_path: Path, monkeypatch) -> None:
    monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)

    class IdleStore(ManagedWatchStore):
        def __init__(self, path: Path):
            super().__init__(path)
            self.reloads = 0
            self.lists = 0

        def maybe_reload(self) -> bool:
            self.reloads += 1
            return False

        def list_watches(self):
            self.lists += 1
            return super().list_watches()

    class CountingRuntimeStore(WatchRuntimeStateStore):
        def __init__(self) -> None:
            self.writes = 0

        def write(self, payload: dict) -> None:
            self.writes += 1

        def load(self) -> dict:
            return {"watches": {}}

    store = IdleStore(tmp_path / "watches.json")
    runtime_store = CountingRuntimeStore()
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    service._running = True
    service._recovery_pending = False
    service._runtime_state_dirty = False

    async def _run() -> None:
        task = asyncio.create_task(service._watch_store())
        await asyncio.sleep(0.05)
        service._running = False
        task.cancel()
        try:
            await task
        except asyncio.CancelledError:
            pass

    asyncio.run(_run())

    assert store.reloads > 0
    assert store.lists == store.reloads
    assert runtime_store.writes == 0


def test_managed_watch_service_start_reaps_stale_worker_for_deleted_watch(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    identity = _persisted_identity(pid=_WORKER_PID)
    _record_watch_pid(runtime_store, "stale-watch", _WORKER_PID, identity=identity)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    terminated: list[int] = []
    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda pid: pid == _WORKER_PID)
    monkeypatch.setattr("core.watches.inspect_process_identity", lambda pid: _live_identity(pid=pid))
    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda pid, *_args, **_kwargs: terminated.append(pid) or True,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        try:
            assert runtime_store.load()["watches"] == {}
        finally:
            await service.stop()

    asyncio.run(_run())
    assert terminated == [_WORKER_PID]


@pytest.mark.parametrize(
    ("command", "shell_command"),
    [
        ([sys.executable, "-c", "print('watch')"], None),
        (["/tmp/wait.py"], None),
        ([], "python3 wait.py --forever"),
    ],
    ids=["exec", "shebang-interpreter", "shell-wrapper"],
)
def test_managed_watch_service_start_reaps_matching_stale_worker_before_reconcile(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    command: list[str],
    shell_command: str | None,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = _add_recovery_watch(store, command=command, shell_command=shell_command)
    identity = _persisted_identity()
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID, identity=identity)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    events: list[tuple[str, int | None]] = []

    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda pid: pid == _WORKER_PID)
    monkeypatch.setattr(
        "core.watches.inspect_process_identity",
        lambda pid: _live_identity(pid=pid),
    )

    def fake_terminate(pid, _logger, _label, *, expected_identity):
        assert expected_identity == identity
        events.append(("terminate", pid))
        return True

    def fake_reconcile():
        events.append(("reconcile", None))
        return False

    monkeypatch.setattr("core.watches.terminate_process_tree_by_pid", fake_terminate)
    monkeypatch.setattr(service, "reconcile_watches", fake_reconcile)

    async def _run() -> None:
        await _start_watch_service(service)
        await service.stop()

    asyncio.run(_run())

    assert events[0] == ("terminate", _WORKER_PID)
    assert events[1:] and all(event == ("reconcile", None) for event in events[1:])


def test_managed_watch_service_start_does_not_reap_reused_pid(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    command = [sys.executable, "wait.py"]
    watch = _add_recovery_watch(store, command=command)
    identity = _persisted_identity()
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID, identity=identity)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    reconciles = 0

    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda pid: pid == _WORKER_PID)
    monkeypatch.setattr(
        "core.watches.inspect_process_identity",
        lambda pid: _live_identity(pid=pid, create_time=456.0, worker_fingerprint=None),
    )
    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda *_args, **_kwargs: pytest.fail("a reused pid must not be terminated"),
    )

    def fake_reconcile():
        nonlocal reconciles
        reconciles += 1
        return False

    monkeypatch.setattr(service, "reconcile_watches", fake_reconcile)

    async def _run() -> None:
        await _start_watch_service(service)
        await service.stop()

    asyncio.run(_run())

    assert reconciles >= 1
    assert watch.id not in service._recovery_blocked_watch_ids


@pytest.mark.parametrize(
    ("live_fingerprint", "unblocked"),
    [(TEST_FINGERPRINT, ()), (None, ("watch-a",))],
)
def test_blocked_watch_recheck_decides_by_marker_not_a_shifted_birth_time(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    live_fingerprint: str | None,
    unblocked: tuple[str, ...],
) -> None:
    """macOS may report a shifted create_time for the same live worker."""

    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=ManagedWatchStore(tmp_path / "watches.json"),
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    identity = _persisted_identity()
    entry = {
        "pid": _WORKER_PID,
        "process_identity": {
            "pid": identity.pid,
            "create_time": identity.create_time,
            "worker_fingerprint": identity.worker_fingerprint,
        },
    }
    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda pid: True)
    monkeypatch.setattr(
        "core.watches.inspect_process_identity",
        lambda pid: _live_identity(pid=pid, create_time=456.0, worker_fingerprint=live_fingerprint),
    )

    assert service._inspect_recovery_blocked_watches({"watch-a": entry}) == unblocked


def test_managed_watch_service_start_blocks_respawn_when_worker_marker_changed(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    command = [sys.executable, "wait.py"]
    watch = _add_recovery_watch(store, command=command)
    identity = _persisted_identity()
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID, identity=identity)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )

    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda pid: pid == _WORKER_PID)
    monkeypatch.setattr(
        "core.watches.inspect_process_identity",
        lambda pid: _live_identity(
            pid=pid,
            worker_fingerprint=fingerprint_process_marker("other-watch-worker"),
        ),
    )
    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda *_args, **_kwargs: pytest.fail("a changed worker marker must not be terminated"),
    )

    async def _run() -> None:
        await _start_watch_service(service)
        assert watch.id not in service._active_tasks
        assert runtime_store.load()["watches"][watch.id]["process_identity"]["create_time"] == 123.0
        await service.stop()

    asyncio.run(_run())

    assert watch.id in service._recovery_blocked_watch_ids
    assert runtime_store.load()["watches"][watch.id]["pid"] == _WORKER_PID


def test_managed_watch_service_start_blocks_legacy_live_worker_without_identity(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = _add_recovery_watch(store, command=[sys.executable, "wait.py"])
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )

    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda pid: pid == _WORKER_PID)
    monkeypatch.setattr(
        "core.watches.inspect_process_identity",
        lambda _pid: _live_identity(),
    )
    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda *_args, **_kwargs: pytest.fail("a legacy entry must not cause termination"),
    )

    async def _run() -> None:
        await _start_watch_service(service)
        assert watch.id not in service._active_tasks
        await service.stop()

    asyncio.run(_run())

    assert watch.id in service._recovery_blocked_watch_ids
    assert runtime_store.load()["watches"][watch.id]["pid"] == _WORKER_PID


def test_managed_watch_service_start_rejects_overflowing_process_identity(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = _add_recovery_watch(store, command=[sys.executable, "wait.py"])
    runtime_store.write(
        {
            "watches": {
                watch.id: {
                    "running": True,
                    "pid": _WORKER_PID,
                    "started_at": "2026-05-15T00:00:00+00:00",
                    "updated_at": "2026-05-15T00:00:01+00:00",
                    "process_identity": {
                        "pid": _WORKER_PID,
                        "create_time": 10**1000,
                        "worker_fingerprint": TEST_FINGERPRINT,
                    },
                }
            }
        }
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )

    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda pid: pid == _WORKER_PID)
    monkeypatch.setattr(
        "core.watches.inspect_process_identity",
        lambda pid: _live_identity(pid=pid),
    )
    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda *_args, **_kwargs: pytest.fail("a malformed identity must not cause termination"),
    )

    async def _run() -> None:
        await _start_watch_service(service)
        assert service._running is True
        assert watch.id in service._recovery_blocked_watch_ids
        assert watch.id not in service._active_tasks
        await service.stop()

    asyncio.run(_run())


def test_managed_watch_service_start_ignores_dead_recorded_pid(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = _add_recovery_watch(store, command=[sys.executable, "wait.py"])
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    reconciles = 0

    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda _pid: False)
    monkeypatch.setattr("core.watches.process_group_exists", lambda *_args: False)
    monkeypatch.setattr(
        "core.watches.inspect_process_identity",
        lambda _pid: pytest.fail("a dead pid must not have its identity inspected"),
    )
    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda *_args, **_kwargs: pytest.fail("a dead pid must not be terminated"),
    )

    def fake_reconcile():
        nonlocal reconciles
        reconciles += 1
        return False

    monkeypatch.setattr(service, "reconcile_watches", fake_reconcile)

    async def _run() -> None:
        await _start_watch_service(service)
        await service.stop()

    asyncio.run(_run())

    assert reconciles >= 1


def test_managed_watch_service_start_reaps_group_after_leader_exit(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = _add_recovery_watch(store, command=[sys.executable, "wait.py"])
    identity = _persisted_identity()
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID, identity=identity)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    events: list[str] = []

    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda _pid: False)
    monkeypatch.setattr("core.watches.process_group_exists", lambda *_args: True)
    monkeypatch.setattr(
        "core.watches.terminate_process_group_by_pgid",
        lambda *_args, **_kwargs: events.append("terminate-group") or True,
    )
    monkeypatch.setattr(
        "core.watches.inspect_process_identity",
        lambda _pid: pytest.fail("an exited leader must not have its identity inspected"),
    )

    def fake_reconcile() -> bool:
        events.append("reconcile")
        return False

    monkeypatch.setattr(service, "reconcile_watches", fake_reconcile)

    async def _run() -> None:
        await _start_watch_service(service)
        await service.stop()

    asyncio.run(_run())

    assert events[0] == "terminate-group"
    assert events[1:] and all(event == "reconcile" for event in events[1:])


def test_managed_watch_service_does_not_reap_unverified_group_after_leader_exit(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = _add_recovery_watch(store, command=[sys.executable, "wait.py"])
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )

    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda _pid: False)
    monkeypatch.setattr("core.watches.process_group_exists", lambda *_args: True)
    monkeypatch.setattr(
        "core.watches.terminate_process_group_by_pgid",
        lambda *_args, **_kwargs: pytest.fail("an unverified process group must not be terminated"),
    )

    async def _run() -> None:
        await _start_watch_service(service)
        assert watch.id in service._recovery_blocked_watch_ids
        assert watch.id not in service._active_tasks
        await service.stop()

    asyncio.run(_run())


def test_managed_watch_service_start_keeps_recovery_off_event_loop(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=ManagedWatchStore(tmp_path / "watches.json"),
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    def inspect_stale_workers() -> _StaleWorkerRecovery:
        time.sleep(0.15)
        return _StaleWorkerRecovery(True)

    monkeypatch.setattr(service, "_inspect_stale_workers", inspect_stale_workers)
    monkeypatch.setattr(service, "reconcile_watches", lambda: False)

    async def _run() -> None:
        started = time.monotonic()
        service.start()
        assert time.monotonic() - started < 0.05
        startup_task = service._startup_task
        assert startup_task is not None
        await asyncio.sleep(0.01)
        assert not startup_task.done()
        await startup_task
        await service.stop()

    asyncio.run(_run())


def test_hfr_164_watch_restart_replaces_only_its_registration_generation(
    tmp_path: Path,
) -> None:
    class EmptyHandler:
        def scan(self, *, limit, occupied, cursor):  # noqa: ANN001, ANN202
            del limit, occupied, cursor
            return [], False

        async def process(self, item) -> None:  # noqa: ANN001
            raise AssertionError(item)

    async def _run() -> None:
        supervisor = RuntimeWorkSupervisor(reconcile_interval=3600)
        delivery_token = supervisor.register(
            RuntimeWorkLane.SESSION_DELIVERIES,
            EmptyHandler(),
        )
        await supervisor.activate()
        service = ManagedWatchService(
            controller=SimpleNamespace(runtime_work_supervisor=supervisor),
            store=ManagedWatchStore(tmp_path / "watches.json"),
            request_store=TaskExecutionStore(tmp_path / "task_requests"),
            runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
        )

        await _start_watch_service(service)
        first = service._runtime_work_token
        assert first is not None
        assert RuntimeWorkLane.SESSION_DELIVERIES in supervisor._registrations
        await service.stop()
        assert RuntimeWorkLane.WATCH_DEFINITIONS not in supervisor._registrations
        assert supervisor._registrations[RuntimeWorkLane.SESSION_DELIVERIES].token == delivery_token

        await _start_watch_service(service)
        second = service._runtime_work_token
        assert second is not None
        assert second.generation > first.generation
        await service.stop()
        await supervisor.stop()

    asyncio.run(_run())


def test_hfr_176_legacy_watch_probe_only_wakes_watch_definitions(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    notified: list[RuntimeWorkLane] = []
    service = ManagedWatchService(
        controller=SimpleNamespace(
            runtime_work_supervisor=SimpleNamespace(
                notify=lambda *lanes: notified.extend(lanes),
            )
        ),
        store=ManagedWatchStore(tmp_path / "watches.json"),
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    service._running = True
    service._legacy_watch_signature = (1, 1, 1)
    monkeypatch.setattr(watches_module, "_path_signature", lambda _path: (2, 2, 2))

    async def stop_after_tick(_delay: float) -> None:
        service._running = False

    monkeypatch.setattr(watches_module.asyncio, "sleep", stop_after_tick)
    asyncio.run(service._legacy_runtime_work_probe())

    assert notified == [RuntimeWorkLane.WATCH_DEFINITIONS]


def test_hfr_179_watch_store_phases_are_single_flight_and_apply_on_loop(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """HFR-179: managed-watch store work stays single-flight and applies on loop."""

    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=ManagedWatchStore(tmp_path / "watches.json"),
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    service._recovery_pending = False
    service._runtime_state_dirty = False
    scan_entered = threading.Event()
    release_scan = threading.Event()
    guarded_entered = threading.Event()
    scan_threads: list[int] = []
    reconcile_threads: list[int] = []

    def blocking_reload() -> bool:
        scan_threads.append(threading.get_ident())
        scan_entered.set()
        assert release_scan.wait(timeout=1)
        return True

    monkeypatch.setattr(service.store, "maybe_reload", blocking_reload)

    async def _run() -> None:
        loop_thread = threading.get_ident()
        handler = _ManagedWatchRuntimeWorkHandler(service)
        scan = asyncio.create_task(
            asyncio.to_thread(
                handler.scan,
                limit=1,
                occupied=frozenset(),
                cursor=None,
            )
        )
        assert await asyncio.to_thread(scan_entered.wait, 1)

        guarded = asyncio.create_task(
            service._watch_store_call_async(
                "watch-a",
                "guarded-test",
                lambda: guarded_entered.set() or True,
                guarded=True,
            )
        )
        await asyncio.sleep(0)
        assert not guarded_entered.is_set()
        release_scan.set()
        items, has_more = await scan
        assert not has_more
        assert await guarded is True

        def reconcile(watches) -> bool:  # noqa: ANN001
            del watches
            reconcile_threads.append(threading.get_ident())
            return False

        monkeypatch.setattr(service, "reconcile_watches", reconcile)
        assert await handler.process(items[0]) is True
        assert scan_threads and scan_threads[0] != loop_thread
        assert reconcile_threads == [loop_thread]

    asyncio.run(_run())


@pytest.mark.parametrize("backend", ["file", "sqlite"])
@pytest.mark.parametrize("ordering", ["before_scan", "after_scan", "legacy"])
def test_hfr_485_watch_reconciles_after_another_reader_consumes_reload(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    backend: str,
    ordering: str,
) -> None:
    """HFR-485: Issue #1995: a refreshed mirror is not an acknowledged scheduling change."""
    monkeypatch.setattr(watches_module, "_publish_watch_definitions_updated", lambda: None)
    monkeypatch.setattr(watches_module, "WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
    reader = ManagedWatchStore(tmp_path / "watches.json")
    writer = ManagedWatchStore(tmp_path / "watches.json")
    sqlite_stores = []
    if backend == "sqlite":
        for store in (reader, writer):
            store._sqlite = SQLiteBackgroundTaskStore(tmp_path / "isolated.sqlite")
            sqlite_stores.append(store._sqlite)
            store.load()
    reader.maybe_reload()
    assert reader.maybe_reload() is False
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=reader,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    service._recovery_pending = False
    handler = _ManagedWatchRuntimeWorkHandler(service)
    started: list[str] = []
    runtime_writes: list[dict] = []
    write_runtime = service.runtime_store.write

    def record_runtime_write(payload: dict) -> None:
        runtime_writes.append(payload)
        write_runtime(payload)

    monkeypatch.setattr(service.runtime_store, "write", record_runtime_write)

    async def _run() -> None:
        new_started = asyncio.Event()
        release_workers = asyncio.Event()
        cleanup_started = asyncio.Event()
        finish_cleanup = asyncio.Event()

        async def worker(watch_id: str) -> None:
            started.append(watch_id)
            if watch_id == "new-watch":
                new_started.set()
            try:
                await release_workers.wait()
            finally:
                if watch_id == "new-watch":
                    cleanup_started.set()
                    await finish_cleanup.wait()

        monkeypatch.setattr(service, "_run_watch", worker)

        async def scan():
            items, _ = await asyncio.to_thread(
                handler.scan, limit=1, occupied=frozenset(), cursor=None,
            )
            return items[0]

        service._running = True
        legacy_task = None
        try:
            writer.upsert_watch(watches_module.ManagedWatch(
                id="existing-watch", name="Existing", session_key="",
            ))
            assert await handler.process(await scan()) is True
            await asyncio.sleep(0)
            assert started == ["existing-watch"]
            original_task = service._active_tasks["existing-watch"]

            # Also cover a definition arriving after scan captured its old snapshot.
            stale_item = await scan() if ordering == "after_scan" else None
            writer.upsert_watch(watches_module.ManagedWatch(
                id="new-watch", name="新 Watch", session_key="",
            ))
            assert await service._watch_store_call_async(
                "existing-watch", "reload before follow-up fence", reader.maybe_reload,
            ) is True
            assert reader.get_watch("new-watch") is not None
            assert reader.maybe_reload() is False

            if ordering == "legacy":
                legacy_task = asyncio.create_task(service._watch_store())
            else:
                if stale_item is not None:
                    assert await handler.process(stale_item) is True
                current = await scan()
                assert {watch.id for watch in current.observation["watches"]} == {
                    "existing-watch", "new-watch",
                }
                assert await handler.process(current) is True
            await asyncio.wait_for(new_started.wait(), timeout=1)

            writes_before_idle = len(runtime_writes)
            for _ in range(3):
                assert await handler.process(await scan()) is True
            await asyncio.sleep(0)
            assert started == ["existing-watch", "new-watch"]
            assert service._active_tasks["existing-watch"] is original_task
            assert len(runtime_writes) == writes_before_idle

            # A consumed disabling change must still retire the existing owner.
            writer.set_enabled("new-watch", False)
            assert await service._watch_store_call_async(
                "existing-watch", "reload", reader.maybe_reload,
            ) is True
            new_task = service._active_tasks["new-watch"]
            assert await handler.process(await scan()) is True
            await asyncio.wait_for(cleanup_started.wait(), timeout=1)
            writes_before_cleanup = len(runtime_writes)
            for _ in range(3):
                assert await handler.process(await scan()) is True
            assert not new_task.done()
            assert len(runtime_writes) == writes_before_cleanup
            finish_cleanup.set()
            await asyncio.gather(new_task, return_exceptions=True)
            assert new_task.cancelled()
            assert service._active_tasks["existing-watch"] is original_task
            writer.set_enabled("new-watch", True)
            new_started.clear()
            assert await handler.process(await scan()) is True
            await asyncio.wait_for(new_started.wait(), timeout=1)
            assert started == ["existing-watch", "new-watch", "new-watch"]
            assert service._active_tasks["new-watch"] is not new_task
        finally:
            service._running = False
            finish_cleanup.set()
            pending = list(service._active_tasks.values())
            if legacy_task is not None:
                pending.append(legacy_task)
            for task in pending:
                task.cancel()
            await asyncio.gather(*pending, return_exceptions=True)

    try:
        asyncio.run(_run())
    finally:
        for store in sqlite_stores:
            store.close()


def test_hfr_179_watch_wake_during_reconcile_replays_after_owner_release(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    supervisor = RuntimeWorkSupervisor(reconcile_interval=3600)
    service = ManagedWatchService(
        controller=SimpleNamespace(runtime_work_supervisor=supervisor),
        store=ManagedWatchStore(tmp_path / "watches.json"),
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    service._recovery_pending = False
    service._runtime_state_dirty = True
    reload_calls = 0
    persist_entered = asyncio.Event()
    release_persist = asyncio.Event()
    second_reload = threading.Event()

    def reload_store() -> bool:
        nonlocal reload_calls
        reload_calls += 1
        if reload_calls == 2:
            second_reload.set()
        return False

    async def persist_runtime_state() -> None:
        persist_entered.set()
        await release_persist.wait()
        service._runtime_state_dirty = False

    monkeypatch.setattr(service.store, "maybe_reload", reload_store)
    monkeypatch.setattr(service, "_persist_runtime_state", persist_runtime_state)

    async def _run() -> None:
        supervisor.register(
            RuntimeWorkLane.WATCH_DEFINITIONS,
            _ManagedWatchRuntimeWorkHandler(service),
        )
        await supervisor.activate()
        await asyncio.wait_for(persist_entered.wait(), 1)

        supervisor.notify(RuntimeWorkLane.WATCH_DEFINITIONS)
        for _ in range(10):
            await asyncio.sleep(0)
        assert reload_calls == 1

        release_persist.set()
        assert await asyncio.to_thread(second_reload.wait, 1)
        assert reload_calls == 2
        await supervisor.stop()

    asyncio.run(_run())


def test_hfr_179_blocked_watch_recovery_arms_generation_scoped_recheck(
    tmp_path: Path,
) -> None:
    delayed: list[tuple[object, float]] = []
    supervisor = SimpleNamespace(
        notify_after=lambda token, delay: delayed.append((token, delay)),
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(runtime_work_supervisor=supervisor),
        store=ManagedWatchStore(tmp_path / "watches.json"),
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    token = object()
    service._runtime_work_token = token  # type: ignore[assignment]
    service._recovery_pending = False
    service._runtime_state_dirty = False
    service._recovery_blocked_watch_ids.add("watch-a")
    handler = _ManagedWatchRuntimeWorkHandler(service)
    item = RuntimeWorkItem(
        "managed-watches",
        {
            "recovery": _StaleWorkerRecovery(True),
            "unblocked": (),
            "watches": (),
            "store_error": None,
        },
        rearm_after_process=False,
    )

    assert asyncio.run(handler.process(item)) is True
    assert delayed == [(token, watches_module.WATCH_RECONCILE_INTERVAL_SECONDS)]


def test_hfr_179_pending_watch_recovery_uses_watch_lane_cadence(
    tmp_path: Path,
) -> None:
    delayed: list[tuple[object, float]] = []
    supervisor = SimpleNamespace(
        notify_after=lambda token, delay: delayed.append((token, delay)),
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(runtime_work_supervisor=supervisor),
        store=ManagedWatchStore(tmp_path / "watches.json"),
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    token = object()
    service._runtime_work_token = token  # type: ignore[assignment]
    handler = _ManagedWatchRuntimeWorkHandler(service)
    item = RuntimeWorkItem(
        "managed-watches",
        {
            "recovery": _StaleWorkerRecovery(False),
            "unblocked": (),
            "watches": (),
            "store_error": None,
        },
        rearm_after_process=False,
    )

    assert asyncio.run(handler.process(item)) is True
    assert service._recovery_pending is True
    assert delayed == [(token, watches_module.WATCH_RECONCILE_INTERVAL_SECONDS)]


def test_hfr_179_paused_watch_store_keeps_probing_and_resumes(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    caplog: pytest.LogCaptureFixture,
) -> None:
    """A store outage pauses the watch lane; it must not end it.

    Consecutive read errors pause reconciliation: the lane drops its fast retry
    and reports the pause once. Every later scan still probes the store, so the
    lane reconciles again on the first clean read. The old one-way fuse stopped
    reading for good, and Watches created after a brief "unable to open database
    file" episode never started until the service restarted.
    """

    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=ManagedWatchStore(tmp_path / "watches.json"),
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    service._recovery_pending = False
    service._runtime_state_dirty = False
    watch = service.store.add_watch(**_WATCH_FIXTURE_PAYLOAD)
    healthy = False
    reads = 0
    reload = service.store.maybe_reload

    def flaky_read() -> bool:
        nonlocal reads
        reads += 1
        if not healthy:
            raise OSError("unable to open database file")
        return reload()

    reconciled: list[list[str]] = []

    def record_reconcile(watches) -> bool:
        reconciled.append([item.id for item in watches])
        return False

    persisted = 0

    async def persist_runtime_state() -> None:
        nonlocal persisted
        persisted += 1
        service._runtime_state_dirty = False

    monkeypatch.setattr(service.store, "maybe_reload", flaky_read)
    monkeypatch.setattr(service, "reconcile_watches", record_reconcile)
    monkeypatch.setattr(service, "_persist_runtime_state", persist_runtime_state)
    handler = _ManagedWatchRuntimeWorkHandler(service)
    pause_after = watches_module.WATCH_STORE_RECONCILE_PAUSE_FAILURES

    async def scan_and_process() -> bool:
        items, _has_more = await asyncio.to_thread(
            handler.scan,
            limit=1,
            occupied=frozenset(),
            cursor=None,
        )
        return await handler.process(items[0])

    async def _run() -> None:
        nonlocal healthy
        outcomes = [await scan_and_process() for _ in range(pause_after)]
        # ``False`` asks the supervisor for a fast retry; the pause stops asking.
        assert outcomes == [False] * (pause_after - 1) + [True]

        service._runtime_state_dirty = True
        assert await scan_and_process() is True, "a paused lane must not fast-retry"
        assert reads == pause_after + 1, "the paused lane stopped probing the store"
        assert persisted == 1, "runtime state must still persist while paused"
        assert reconciled == []

        healthy = True
        assert await scan_and_process() is True
        assert reconciled == [[watch.id]], "the lane did not resume once the store read cleanly"
        assert service._store_reconcile_failures == 0

    with caplog.at_level(logging.DEBUG, logger="core.watches"):
        asyncio.run(_run())

    errors = [record for record in caplog.records if record.levelno >= logging.ERROR]
    assert len(errors) == 1 and "Pausing managed watch reconciliation" in errors[0].message
    assert any("reconciliation resumed" in record.message for record in caplog.records)


def test_watch_runtime_state_persistence_failure_rearms_maintenance(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=ManagedWatchStore(tmp_path / "watches.json"),
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
    )
    service._recovery_pending = False
    service._runtime_state_dirty = True

    async def _persist_but_remain_dirty() -> None:
        service._runtime_state_dirty = True

    monkeypatch.setattr(service, "_persist_runtime_state", _persist_but_remain_dirty)
    handler = _ManagedWatchRuntimeWorkHandler(service)
    item = RuntimeWorkItem(
        "managed-watches",
        {
            "recovery": _StaleWorkerRecovery(True),
            "unblocked": (),
            "watches": (),
        },
        rearm_after_process=False,
    )

    assert asyncio.run(handler.process(item)) is False
    assert service._runtime_state_dirty is True


def test_managed_watch_service_start_retries_unreadable_runtime_before_reconcile(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    caplog: pytest.LogCaptureFixture,
) -> None:
    class UnreadableRuntimeStore(WatchRuntimeStateStore):
        def __init__(self) -> None:
            self.writes = 0
            self.readable = False

        def load_for_recovery(self) -> dict:
            if not self.readable:
                raise OSError("runtime state is temporarily unavailable")
            return {"watches": {}}

        def write(self, payload: dict) -> None:
            self.writes += 1

    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = UnreadableRuntimeStore()
    monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    reconciles = 0

    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda *_args, **_kwargs: pytest.fail("unreadable state must not cause termination"),
    )

    def fake_reconcile():
        nonlocal reconciles
        reconciles += 1
        return False

    monkeypatch.setattr(service, "reconcile_watches", fake_reconcile)

    async def _run() -> None:
        await _start_watch_service(service)
        assert service._recovery_pending is True
        assert reconciles == 0
        assert runtime_store.writes == 0
        runtime_store.readable = True
        for _ in range(100):
            if reconciles:
                break
            await asyncio.sleep(0.01)
        await service.stop()

    with caplog.at_level("WARNING"):
        asyncio.run(_run())

    assert reconciles >= 1
    assert runtime_store.writes > 0
    assert "Unable to read prior watch runtime state" in caplog.text


def test_managed_watch_service_start_retries_malformed_runtime_before_reconcile(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    caplog: pytest.LogCaptureFixture,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_path = tmp_path / "watch_runtime.json"
    runtime_path.write_text("{not-json", encoding="utf-8")
    runtime_store = WatchRuntimeStateStore(runtime_path)
    monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    reconciles = 0

    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda *_args, **_kwargs: pytest.fail("malformed state must not cause termination"),
    )

    def fake_reconcile():
        nonlocal reconciles
        reconciles += 1
        return False

    monkeypatch.setattr(service, "reconcile_watches", fake_reconcile)

    async def _run() -> None:
        await _start_watch_service(service)
        assert service._recovery_pending is True
        assert reconciles == 0
        runtime_path.write_text('{"watches": {}}', encoding="utf-8")
        for _ in range(100):
            if reconciles:
                break
            await asyncio.sleep(0.01)
        await service.stop()

    with caplog.at_level("WARNING"):
        asyncio.run(_run())

    assert reconciles >= 1
    assert "Unable to read prior watch runtime state" in caplog.text


def test_managed_watch_service_start_retries_unavailable_watch_list_before_reconcile(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    caplog: pytest.LogCaptureFixture,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = _add_recovery_watch(store, command=[sys.executable, "wait.py"])
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    list_calls = 0
    available = {"value": False}
    monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)

    def failing_list():
        nonlocal list_calls
        list_calls += 1
        if not available["value"]:
            raise RuntimeError("watch definitions are temporarily unavailable")
        return store.list_watches()

    monkeypatch.setattr(store, "list_watches_for_recovery", failing_list)
    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda *_args, **_kwargs: pytest.fail("an unavailable watch list must not cause termination"),
    )

    async def fake_run_watch(_watch_id: str) -> None:
        await asyncio.Future()

    monkeypatch.setattr(service, "_run_watch", fake_run_watch)

    async def _run() -> None:
        await _start_watch_service(service)
        assert service._running is True
        assert service._recovery_pending is True
        assert watch.id not in service._active_tasks
        available["value"] = True
        for _ in range(100):
            if watch.id in service._active_tasks:
                break
            await asyncio.sleep(0.01)
        assert watch.id in service._active_tasks
        await service.stop()

    with caplog.at_level("WARNING"):
        asyncio.run(_run())

    assert list_calls >= 2
    assert service._store_reconcile_failures == 0
    assert "Unable to read managed watch definitions" in caplog.text


def test_managed_watch_service_start_does_not_reap_with_malformed_watch_store(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
    caplog: pytest.LogCaptureFixture,
) -> None:
    watches_path = tmp_path / "watches.json"
    original_store = ManagedWatchStore(watches_path)
    watch = _add_recovery_watch(original_store, command=[sys.executable, "wait.py"])
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID)
    watches_path.write_text("{not-json", encoding="utf-8")

    with caplog.at_level("WARNING"):
        store = ManagedWatchStore(watches_path)
        service = ManagedWatchService(
            controller=SimpleNamespace(),
            store=store,
            request_store=TaskExecutionStore(tmp_path / "task_requests"),
            runtime_store=runtime_store,
        )
        monkeypatch.setattr(
            "core.watches.runtime.pid_alive",
            lambda _pid: pytest.fail("a malformed watch store must not cause pid inspection"),
        )
        monkeypatch.setattr(
            "core.watches.terminate_process_tree_by_pid",
            lambda *_args, **_kwargs: pytest.fail("a malformed watch store must not cause termination"),
        )

        async def _run() -> None:
            await _start_watch_service(service)
            assert service._active_tasks == {}
            await service.stop()

        asyncio.run(_run())

    assert "Unable to read managed watch definitions" in caplog.text


def test_managed_watch_service_start_preserves_worker_state_when_reap_fails(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    command = [sys.executable, "wait.py"]
    watch = _add_recovery_watch(store, command=command)
    identity = _persisted_identity()
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID, identity=identity)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda pid: pid == _WORKER_PID)
    monkeypatch.setattr("core.watches.inspect_process_identity", lambda pid: _live_identity(pid=pid))
    monkeypatch.setattr("core.watches.terminate_process_tree_by_pid", lambda *_args, **_kwargs: False)

    async def _run() -> None:
        await _start_watch_service(service)
        assert service._active_tasks == {}
        assert runtime_store.load()["watches"][watch.id]["pid"] == _WORKER_PID
        await service.stop()

    asyncio.run(_run())

    assert watch.id in service._recovery_blocked_watch_ids
    assert runtime_store.load()["watches"][watch.id]["pid"] == _WORKER_PID


def test_managed_watch_service_periodically_unblocks_after_stale_worker_exits(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    command = [sys.executable, "wait.py"]
    watch = _add_recovery_watch(store, command=command)
    identity = _persisted_identity()
    _record_watch_pid(runtime_store, watch.id, _WORKER_PID, identity=identity)
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )
    worker = {"alive": True}

    monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
    monkeypatch.setattr("core.watches.runtime.pid_alive", lambda _pid: worker["alive"])
    monkeypatch.setattr("core.watches.process_group_exists", lambda *_args: False)
    monkeypatch.setattr("core.watches.inspect_process_identity", lambda pid: _live_identity(pid=pid))
    monkeypatch.setattr("core.watches.terminate_process_tree_by_pid", lambda *_args, **_kwargs: False)

    async def fake_run_watch(_watch_id: str) -> None:
        await asyncio.Future()

    monkeypatch.setattr(service, "_run_watch", fake_run_watch)

    async def _run() -> None:
        await _start_watch_service(service)
        assert watch.id in service._recovery_blocked_watch_ids
        assert watch.id not in service._active_tasks

        worker["alive"] = False
        for _ in range(100):
            if watch.id in service._active_tasks:
                break
            await asyncio.sleep(0.01)

        assert watch.id not in service._recovery_blocked_watch_ids
        assert watch.id not in service._unreaped_runtime_entries
        assert watch.id in service._active_tasks
        await service.stop()

    asyncio.run(_run())


def test_managed_watch_service_start_without_prior_runtime_state_reconciles_normally(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    store = ManagedWatchStore(tmp_path / "watches.json")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    watch = _add_recovery_watch(store, command=[sys.executable, "wait.py"])
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=TaskExecutionStore(tmp_path / "task_requests"),
        runtime_store=runtime_store,
    )

    monkeypatch.setattr(
        store,
        "list_watches_for_recovery",
        lambda: pytest.fail("empty runtime state must not reload watch definitions"),
    )
    monkeypatch.setattr(
        "core.watches.runtime.pid_alive",
        lambda _pid: pytest.fail("empty runtime state must not inspect any pids"),
    )
    monkeypatch.setattr(
        "core.watches.inspect_process_identity",
        lambda _pid: pytest.fail("empty runtime state must not inspect any processes"),
    )
    monkeypatch.setattr(
        "core.watches.terminate_process_tree_by_pid",
        lambda *_args, **_kwargs: pytest.fail("empty runtime state must not cause termination"),
    )

    async def fake_run_watch(_watch_id: str) -> None:
        await asyncio.Future()

    monkeypatch.setattr(service, "_run_watch", fake_run_watch)

    async def _run() -> None:
        await _start_watch_service(service)
        assert watch.id in service._active_tasks
        await service.stop()

    asyncio.run(_run())


def test_managed_watch_service_retries_transient_reconcile_errors(tmp_path: Path, monkeypatch) -> None:
    monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)

    class TransientListStore(ManagedWatchStore):
        def __init__(self, path: Path):
            super().__init__(path)
            self.failures_remaining = 2

        def list_watches(self):
            if self.failures_remaining > 0:
                self.failures_remaining -= 1
                raise RuntimeError("database is locked")
            return super().list_watches()

    store = TransientListStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        service._running = True
        task = asyncio.create_task(service._watch_store())
        for _ in range(100):
            if store.failures_remaining == 0 and service._store_reconcile_failures == 0:
                break
            await asyncio.sleep(0.05)
        service._running = False
        task.cancel()
        try:
            await task
        except asyncio.CancelledError:
            pass

    asyncio.run(_run())

    assert service._store_reconcile_failures == 0


def test_managed_watch_service_start_retries_initial_reconcile_error(tmp_path: Path) -> None:
    class FailingListStore(ManagedWatchStore):
        def list_watches(self):
            raise RuntimeError("database disk image is malformed")

    store = FailingListStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        assert 1 <= service._store_reconcile_failures < 3
        await service.stop()

    asyncio.run(_run())


def test_managed_watch_service_ignores_runtime_state_write_failure(tmp_path: Path) -> None:
    class FailingRuntimeStore(WatchRuntimeStateStore):
        def __init__(self) -> None:
            self.writes = 0

        def write(self, payload: dict) -> None:
            self.writes += 1
            raise RuntimeError("database disk image is malformed")

        def load(self) -> dict:
            return {"watches": {}}

        def load_for_recovery(self) -> dict:
            return {"watches": {}}

    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = FailingRuntimeStore()
    watch = store.add_watch(
        name="Runtime failure",
        session_key="slack::channel::C123",
        command=[sys.executable, "-c", "print('done')"],
        shell_command=None,
        prefix="Finished.",
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        await _start_watch_service(service)
        for _ in range(100):
            if watch.id not in service._active_tasks:
                break
            await asyncio.sleep(0.02)
        await service.stop()

    asyncio.run(_run())

    saved = store.get_watch(watch.id)
    assert saved is not None
    assert saved.enabled is False
    assert runtime_store.writes > 0


#: The ``existing`` probe ``_upsert_definition`` runs before its guarded UPDATE.
#: The competing reclaim is committed when THIS read completes, which puts it
#: exactly inside the window the guard exists for: after the supervisor decided
#: what to write, before the write takes the lock.
_DEFINITION_EXISTS_SELECT = (
    "SELECT run_definitions.id FROM run_definitions WHERE run_definitions.id = ? LIMIT ? OFFSET ?"
)


def _bare_watch_session_row(*, workdir: Path, anchor: str = "slack_C123") -> str:
    """A real Session row for a watch to be bound to."""
    from config import paths as config_paths
    from storage.agent_session_rows import create_agent_session_row
    from storage.db import create_sqlite_engine

    engine = create_sqlite_engine(config_paths.get_sqlite_state_path())
    try:
        with engine.begin() as conn:
            return create_agent_session_row(
                conn,
                scope_id=None,
                session_anchor=anchor,
                agent_backend="codex",
                agent_variant="codex",
                model="gpt-5.5-codex",
                native_session_id="codex-native",
                workdir=str(workdir),
                require_workdir=False,
            )
    finally:
        engine.dispose()


def _commit_reclaim_after(engine, session_id: str, *, read: str, reason: str, occurrence: int = 1) -> dict:
    """Commit the REAL ``/new`` reclaim from a genuinely separate connection.

    The watch-side twin of ``_commit_competing_bind_after`` in
    ``tests/test_sqlite_sessions_store.py``: hooks ``after_cursor_execute`` on the
    engine the code under test uses, and when ``read`` completes opens its own
    engine, runs ``reclaim_bound_definitions`` and COMMITS. Control returns to the
    supervisor mid-write, so its next statement runs against a database another
    writer has already changed. Fires once; the returned dict records it, so a
    rendered-SQL drift shows up as "never raced" instead of a vacuous pass.

    ``occurrence`` selects WHICH guarded write to race. One ``_run_watch`` iteration
    runs several of them (``mark_cycle_start``, then ``mark_cycle_result``), and each
    emits ``read`` once, so ``occurrence=2`` lands the reclaim inside the RESULT
    stamp's write window -- after the start stamp has already been accepted. That is
    the window HFR-267 is about: the cycle legitimately ran, and the teardown arrives
    while its outcome is being recorded.
    """
    from sqlalchemy import event

    from config import paths as config_paths
    from storage.db import create_sqlite_engine
    from storage.session_reclaim import RECLAIM_PAUSE, reclaim_bound_definitions

    state: dict = {"fired": 0, "seen": 0, "summary": None}

    @event.listens_for(engine, "after_cursor_execute")
    def _race(conn, cursor, statement, parameters, context, executemany) -> None:  # noqa: ANN001
        if state["fired"] or " ".join(statement.split()) != read:
            return
        state["seen"] += 1
        if state["seen"] < occurrence:
            return
        state["fired"] += 1
        other = create_sqlite_engine(config_paths.get_sqlite_state_path())
        try:
            with other.begin() as other_conn:
                state["summary"] = reclaim_bound_definitions(
                    other_conn, session_id, mode=RECLAIM_PAUSE, reason=reason
                )
        finally:
            other.dispose()

    return state


def test_run_watch_stops_when_its_start_stamp_loses_to_a_reclaim(tmp_path: Path) -> None:
    """HFR-263 — a refused ``mark_cycle_start`` must stop the cycle, not be discarded.

    THE PRODUCTION STORY. A ``forever`` watch is bound to a Session. The user types
    ``/new`` in that channel: the teardown deletes the session row and
    ``reclaim_bound_definitions(mode='pause')`` pauses the watch and stamps its
    settings snapshot, and ``/new`` replies "1 watch paused". The supervisor loop was
    already past its own ``maybe_reload``, so it holds the pre-teardown mirror.

    THE CONSUMING END WAS MISSING. HFR-261 made ``mark_cycle_start`` a guarded
    full-row write that correctly REFUSES the stale payload and returns ``False`` --
    but ``_watch_store_call`` was ``callback(); return True``, discarding the
    callback's own return value. The refusal was reported to the loop as success, so
    the cycle went on to spawn its waiter subprocess and enqueue a hook against the
    definition the database had just torn down: the user is told the watch is paused
    while it runs a command and delivers a prompt into a dead session.

    Driven through the REAL ``_run_watch``, with the reclaim committed from a second
    connection INSIDE the write window. ``_run_cycle`` is the only double, and it
    returns a SUCCESS result on purpose: on the unfixed wrapper that is what produces
    the enqueued hook this test forbids.
    """
    from storage.session_reclaim import SESSION_SETTINGS_SNAPSHOT_KEY

    reason = "the bound agent session was cleared"
    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test needs the SQLite-backed store; the guard lives there"
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    session_id = _bare_watch_session_row(workdir=tmp_path)
    watch = store.add_watch(
        name="Watch CI",
        session_key="",
        session_id=session_id,
        session_policy="existing",
        command=[sys.executable, "-c", "print('event')"],
        shell_command=None,
        prefix="CI is green.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
        metadata={"origin": "cli"},
    )

    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    service._running = True
    service._requires_service_lease = False

    cycles: list[str] = []

    async def _spy_cycle(watch_arg, *, timeout_seconds):  # noqa: ANN001
        cycles.append(watch_arg.id)
        return _CycleResult(exit_code=0, stdout="ci is green", stderr="", timed_out=False)

    service._run_cycle = _spy_cycle  # type: ignore[method-assign]

    race = _commit_reclaim_after(
        store._sqlite.engine, session_id, read=_DEFINITION_EXISTS_SELECT, reason=reason
    )

    asyncio.run(service._run_watch(watch.id))

    assert race["fired"] == 1, (
        "the competing reclaim never landed inside the write window, so this test "
        "proved nothing; the rendered SQL of the guarded upsert's existence probe drifted"
    )
    assert race["summary"] == {"paused": 1, "deleted": 0, "snapshotted": 1}, (
        f"the reclaim itself did not land ({race['summary']!r})"
    )

    assert cycles == [], (
        "the cycle ran its command after the store refused the start stamp; the "
        "waiter subprocess is spawned for a watch the teardown already paused"
    )
    assert request_store.list_pending() == [], (
        "a hook was enqueued for a refused cycle; the prompt is delivered into the "
        "session /new just tore down, after /new told the user the watch was paused"
    )

    stored = ManagedWatchStore().get_watch(watch.id)
    assert stored is not None
    assert stored.enabled is False, (
        "the refused start stamp re-enabled the watch the reclaim paused"
    )
    assert stored.last_error == reason, "the reclaim's pause reason was overwritten"
    assert stored.last_started_at is None, (
        "the start stamp partially landed — a lost compare-and-set must change NOTHING"
    )
    assert stored.last_exit_code is None and stored.last_finished_at is None, (
        "a success-shaped cycle result was recorded for a cycle that never ran"
    )
    assert SESSION_SETTINGS_SNAPSHOT_KEY in stored.metadata, (
        "the reclaim's settings snapshot was replaced by the pre-teardown metadata"
    )
    # HFR-271's rule: the refusal is only proven when the DURABLE row above and the LIVE
    # store the service is still running on say the same thing. ``store`` is the object
    # ``_run_watch`` mutated before the refused stamp, and the one ``reconcile_watches``
    # reads next.
    live = store.get_watch(watch.id)
    assert live is not None and live.to_dict() == stored.to_dict(), (
        "the start stamp was refused and the live store kept it: it serves "
        f"enabled={None if live is None else live.enabled!r} "
        f"last_started_at={None if live is None else live.last_started_at!r} while the "
        f"row says enabled={stored.enabled!r} last_started_at={stored.last_started_at!r}"
    )


#: Sentinel ``cwd``: replaced with a path under ``tmp_path`` that is deliberately
#: never created, so ``_missing_watch_cwd_error`` fires on the real filesystem.
_MISSING_CWD = "<missing-cwd>"

#: One row per ``_run_watch`` branch that enqueues a completion hook. ALL FIVE of
#: them: every ``_commit_cycle_result`` call site that can produce a hook is a row
#: here, so a branch cannot be added to production without a control below.
#:
#: ``cycles`` is what the doubled ``_run_cycle`` returns, one entry per call.
#: ``occurrence`` is which guarded definition write to land the reclaim inside --
#: every ``mark_cycle_start`` and ``mark_cycle_result`` emits the existence probe
#: once, in order, so the number identifies the RESULT stamp of the branch under
#: test. ``expect_hook`` is a fragment of the prompt the branch delivers, which is
#: what proves the control run reached that branch instead of some other one.
#: ``expect_cycle`` is False for the one branch that stops before the waiter spawns.
_HOOK_BRANCHES: dict[str, dict] = {
    # exit 0 -> the event fired; the watch delivers the waiter's stdout.
    "success": {
        "overrides": {"mode": "once", "lifetime_timeout_seconds": 0},
        "cycles": [_CycleResult(exit_code=0, stdout="ci is green", stderr="", timed_out=False)],
        "occurrence": 2,
        "expect_hook": "ci is green",
        "expect_exit_code": 0,
    },
    # the waiter overran its per-cycle timeout and 124 is not a retry code.
    "cycle_timeout": {
        "overrides": {"mode": "forever", "lifetime_timeout_seconds": 0},
        "cycles": [_CycleResult(exit_code=124, stdout="", stderr="", timed_out=True)],
        "occurrence": 2,
        "expect_hook": "the waiter timed out",
        "expect_exit_code": 124,
    },
    # the waiter exited non-zero with a code outside ``retry_exit_codes``.
    "terminal_failure": {
        "overrides": {"mode": "forever", "lifetime_timeout_seconds": 0},
        "cycles": [_CycleResult(exit_code=2, stdout="", stderr="boom", timed_out=False)],
        "occurrence": 2,
        "expect_hook": "exited with code 2",
        "expect_exit_code": 2,
    },
    # The supervisor's own deadline. The shared fixture arms the lifetime only after
    # cycle 1's retry result has landed, then expires it after iteration 2 reloads the
    # live mirror. The lifetime branch runs BEFORE ``mark_cycle_start``. Writes in order:
    # start #1, retry result #2, lifetime result #3.
    "lifetime_expiry": {
        "overrides": {"mode": "forever", "lifetime_timeout_seconds": 0},
        "cycles": [_CycleResult(exit_code=75, stdout="", stderr="not yet", timed_out=False)],
        "occurrence": 3,
        "expect_hook": "reached its lifetime timeout",
        "expect_exit_code": 124,
    },
    # the definition's working directory is gone (a deleted worktree). The check runs
    # right AFTER ``mark_cycle_start`` lands and before the waiter is spawned, so this
    # is the one hook-producing branch that never runs a cycle. Writes in order:
    # start #1, missing-cwd result #2.
    "missing_cwd": {
        "overrides": {"mode": "forever", "lifetime_timeout_seconds": 0, "cwd": _MISSING_CWD},
        "cycles": [],
        "occurrence": 2,
        "expect_hook": "working directory is no longer available",
        "expect_exit_code": 1,
        "expect_cycle": False,
    },
}


def _assert_branch_was_reached(branch: str, calls: list[str]) -> None:
    """The scenario is only wired if the branch under test was actually taken.

    Four branches are reached by running a cycle. ``missing_cwd`` is reached by NOT
    running one -- a spawned waiter there would mean the run took a different path --
    and the guarded result write the callers assert on is what proves it arrived.
    """

    if _HOOK_BRANCHES[branch].get("expect_cycle", True):
        assert calls, f"the {branch} branch never ran a cycle, so the scenario is not wired"
    else:
        assert calls == [], (
            f"the {branch} branch spawned a waiter, so the run did not take the "
            f"missing-cwd path at all: {calls!r}"
        )


def _hook_branch_service(tmp_path: Path, branch: str, *, request_store=None) -> tuple:
    """Build a real store/service/watch for one ``_HOOK_BRANCHES`` row.

    ``request_store`` defaults to a file-backed outbox, which is enough for the
    ordering tests. Pass ``TaskExecutionStore()`` for the SQLite outbox — the real
    ``agent_runs`` ledger, and the only backend that can share a transaction with the
    definition stamp (HFR-269).
    """

    spec = _HOOK_BRANCHES[branch]
    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test needs the SQLite-backed store; the guard lives there"
    if request_store is None:
        request_store = TaskExecutionStore(tmp_path / f"requests_{branch}")
    runtime_store = WatchRuntimeStateStore(tmp_path / f"runtime_{branch}.json")
    session_id = _bare_watch_session_row(workdir=tmp_path, anchor=f"slack_C_{branch}")
    kwargs = {
        "name": f"Watch {branch}",
        "session_key": "",
        "session_id": session_id,
        "session_policy": "existing",
        "command": [sys.executable, "-c", "print('event')"],
        "shell_command": None,
        "prefix": "CI update.",
        "cwd": None,
        "mode": "forever",
        "timeout_seconds": 5,
        "lifetime_timeout_seconds": 0,
        "retry_exit_codes": [75],
        "retry_delay_seconds": 0.01,
        "post_to": None,
        "deliver_key": None,
        "metadata": {"origin": "cli"},
    }
    kwargs.update(spec["overrides"])
    if kwargs.get("cwd") == _MISSING_CWD:
        # Never created on purpose: the production check is a real ``Path.is_dir()``.
        kwargs["cwd"] = str(tmp_path / f"removed_worktree_{branch}")
    watch = store.add_watch(**kwargs)

    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    service._running = True
    service._requires_service_lease = False

    results = list(spec["cycles"])
    calls: list[str] = []

    async def _spy_cycle(watch_arg, *, timeout_seconds):  # noqa: ANN001
        calls.append(watch_arg.id)
        return results[min(len(calls) - 1, len(results) - 1)]

    service._run_cycle = _spy_cycle  # type: ignore[method-assign]
    if branch == "lifetime_expiry":
        # Production reaches _sleep_before_retry only after the guarded retry-result
        # stamp commits. Use that boundary as the condition; no wall-clock wait needed.
        retry_result_landed = False
        wait_for_follow_up_slot = service._wait_for_follow_up_slot

        async def _mark_retry_result_landed(watch_arg, *, lifetime_started):  # noqa: ANN001
            nonlocal retry_result_landed
            assert calls == [watch_arg.id], "lifetime expiry must follow exactly one retry cycle"
            retry_result_landed = True

        async def _expire_after_next_reload(watch_id, *, lifetime_started):  # noqa: ANN001
            slot = await wait_for_follow_up_slot(
                watch_id,
                lifetime_started=lifetime_started,
            )
            if retry_result_landed:
                live_watch = store.get_watch(watch_id)
                assert live_watch is not None
                elapsed = asyncio.get_running_loop().time() - lifetime_started
                assert elapsed > 0, "the retry result landed before the lifetime clock advanced"
                live_watch.lifetime_timeout_seconds = elapsed
            return slot

        service._sleep_before_retry = _mark_retry_result_landed  # type: ignore[method-assign]
        service._wait_for_follow_up_slot = _expire_after_next_reload  # type: ignore[method-assign]
    return store, service, watch, request_store, session_id, calls


@pytest.mark.parametrize("branch", list(_HOOK_BRANCHES))
def test_run_watch_enqueues_the_branch_hook_when_the_result_stamp_lands(tmp_path: Path, branch: str) -> None:
    """HFR-267 control — with no teardown racing it, every branch DOES deliver its hook.

    Without this, the refusal tests below could pass because the branch was never
    reached at all. Here the same wiring, minus the reclaim, must enqueue exactly one
    hook carrying that branch's own text.
    """
    _store, service, watch, request_store, _session_id, calls = _hook_branch_service(tmp_path, branch)

    asyncio.run(service._run_watch(watch.id))

    _assert_branch_was_reached(branch, calls)
    pending = request_store.list_pending()
    assert len(pending) == 1, f"the {branch} branch enqueued {len(pending)} hooks, expected exactly 1"
    assert _HOOK_BRANCHES[branch]["expect_hook"] in (pending[0].prompt or ""), (
        f"the enqueued hook is not the {branch} branch's: {pending[0].prompt!r}"
    )


@pytest.mark.parametrize("branch", list(_HOOK_BRANCHES))
def test_run_watch_enqueues_no_hook_when_the_result_stamp_is_refused(tmp_path: Path, branch: str) -> None:
    """HFR-267 — the guarded stamp must run BEFORE the hook it authorises, in every branch.

    THE PRODUCTION STORY. A watch is bound to a Session and its cycle completes. In the
    same instant the user types ``/new`` in that channel: the teardown deletes the
    session row, ``reclaim_bound_definitions(mode='pause')`` pauses the watch, and
    ``/new`` replies "1 watch paused".

    THE GUARD WAS ASKED TOO LATE. HFR-261 made ``mark_cycle_result`` a guarded
    compare-and-set and HFR-263 taught ``_watch_store_call`` to honour its refusal --
    but all four hook-enqueueing branches ran

        self._enqueue_hook(...)                      # durable request row, now unrecallable
        if not self._watch_store_call(..., guarded=True):
            return                                  # refused; unqueues nothing

    ``enqueue_hook_send`` writes a row a separate drain loop claims and executes, so
    ``return`` after the refusal cancels nothing: the prompt is still delivered into
    the session ``/new`` just deleted, for a definition the database has torn down,
    after the user was told the watch was paused. Respecting the guard's answer is not
    enough -- the effect has to come after the guard.

    Driven through the REAL ``_run_watch``, with the real ``/new`` reclaim committed
    from a second connection INSIDE the result stamp's write window. ``_run_cycle`` is
    the only double, and it returns each branch's own success/timeout/failure shape on
    purpose: on the unfixed ordering that is exactly what produces the hook this test
    forbids.
    """
    from storage.session_reclaim import SESSION_SETTINGS_SNAPSHOT_KEY

    reason = "the bound agent session was cleared"
    store, service, watch, request_store, session_id, calls = _hook_branch_service(tmp_path, branch)

    race = _commit_reclaim_after(
        store._sqlite.engine,
        session_id,
        read=_DEFINITION_EXISTS_SELECT,
        reason=reason,
        occurrence=_HOOK_BRANCHES[branch]["occurrence"],
    )

    asyncio.run(service._run_watch(watch.id))

    assert race["fired"] == 1, (
        f"the competing reclaim never landed inside the {branch} result stamp's write "
        f"window (saw {race['seen']} guarded writes, wanted #{_HOOK_BRANCHES[branch]['occurrence']}), "
        "so this test proved nothing"
    )
    assert race["summary"] == {"paused": 1, "deleted": 0, "snapshotted": 1}, (
        f"the reclaim itself did not land ({race['summary']!r})"
    )
    _assert_branch_was_reached(branch, calls)

    assert request_store.list_pending() == [], (
        f"the {branch} branch enqueued a hook for a cycle result the store REFUSED; the "
        "prompt is delivered into the session /new just tore down, after /new told the "
        "user the watch was paused"
    )

    stored = ManagedWatchStore().get_watch(watch.id)
    assert stored is not None
    assert stored.enabled is False, "the refused result stamp re-enabled the watch the reclaim paused"
    assert stored.last_error == reason, "the reclaim's pause reason was overwritten by the refused stamp"
    assert SESSION_SETTINGS_SNAPSHOT_KEY in stored.metadata, (
        "the reclaim's settings snapshot was replaced by the pre-teardown metadata"
    )
    # HFR-271's rule, on the live half a fresh store cannot see.
    live = service.store.get_watch(watch.id)
    assert live is not None and live.to_dict() == stored.to_dict(), (
        "the result stamp was refused and the live store kept it: it serves "
        f"enabled={None if live is None else live.enabled!r} "
        f"retired_at={None if live is None else live.retired_at!r} while the row says "
        f"enabled={stored.enabled!r} retired_at={stored.retired_at!r}"
    )


#: The outbox row's own INSERT. Matched by prefix so a new ``agent_runs`` column
#: cannot silently stop these tests from racing anything.
_AGENT_RUNS_INSERT = "INSERT INTO AGENT_RUNS"


def _is_agent_runs_insert(statement: str) -> bool:
    return " ".join(statement.split()).upper().startswith(_AGENT_RUNS_INSERT)


def _try_archive_before_the_outbox_insert(engines, session_id: str, *, reason: str) -> dict:
    """Try to commit the REAL archive teardown in the gap before the outbox INSERT.

    Hooks ``before_cursor_execute`` on every engine the code under test might write the
    ``agent_runs`` row through — the watch store's and the request store's, because
    WHICH one carries the INSERT is exactly what this test is about — and when that
    INSERT is about to run, opens its own engine and commits
    ``reclaim_bound_definitions(mode='delete')`` from a genuinely separate connection.

    ``PRAGMA busy_timeout = 0`` is what turns the outcome into a FACT instead of a five
    second sleep: at this instant the result stamp has already been decided, so if the
    stamp was its OWN transaction the database is unlocked and the teardown commits
    immediately; if the stamp and this INSERT are one transaction the write lock is
    already held and SQLite refuses at once with "database is locked". ``blocked``
    records which of the two happened, so "there is no gap" is asserted rather than
    assumed, and a rendered-SQL drift shows up as "never raced".
    """
    from sqlalchemy import event
    from sqlalchemy.exc import OperationalError

    from config import paths as config_paths
    from storage.db import create_sqlite_engine
    from storage.session_reclaim import RECLAIM_DELETE, reclaim_bound_definitions

    state: dict = {"fired": 0, "blocked": None, "summary": None}

    def _race(conn, cursor, statement, parameters, context, executemany) -> None:  # noqa: ANN001
        if state["fired"] or not _is_agent_runs_insert(statement):
            return
        state["fired"] += 1
        other = create_sqlite_engine(config_paths.get_sqlite_state_path())
        try:
            with other.begin() as other_conn:
                other_conn.exec_driver_sql("PRAGMA busy_timeout = 0")
                state["summary"] = reclaim_bound_definitions(
                    other_conn, session_id, mode=RECLAIM_DELETE, reason=reason
                )
            state["blocked"] = False
        except OperationalError as exc:
            # Serialised behind the transaction that holds the stamp: the teardown
            # cannot land in between, which is the property under test.
            state["blocked"] = True
            state["detail"] = str(exc)
        finally:
            other.dispose()

    for engine in engines:
        event.listens_for(engine, "before_cursor_execute")(_race)
    return state


def _fail_the_outbox_insert(engines) -> dict:
    """Make the outbox row's INSERT fail, wherever the code under test writes it."""
    from sqlalchemy import event

    state: dict = {"fired": 0}

    def _boom(conn, cursor, statement, parameters, context, executemany) -> None:  # noqa: ANN001
        if not _is_agent_runs_insert(statement):
            return
        state["fired"] += 1
        raise RuntimeError("outbox write failed: disk I/O error")

    for engine in engines:
        event.listens_for(engine, "before_cursor_execute")(_boom)
    return state


def _hook_branch_service_on_the_run_ledger(tmp_path: Path, branch: str) -> tuple:
    """``_hook_branch_service`` on the SQLite outbox — the real ``agent_runs`` ledger."""

    request_store = TaskExecutionStore()
    assert request_store._sqlite is not None, "this test needs the SQLite outbox; a transaction is the fix"
    store, service, watch, request_store, session_id, calls = _hook_branch_service(
        tmp_path, branch, request_store=request_store
    )
    assert store._sqlite.db_path == request_store._sqlite.db_path, (
        "the definition and the outbox must be in ONE database for a shared transaction "
        "to be possible at all"
    )
    engines = [store._sqlite.engine, request_store._sqlite.engine]
    return store, service, watch, request_store, session_id, calls, engines


@pytest.mark.parametrize("branch", list(_HOOK_BRANCHES))
def test_the_atomic_stamp_and_its_hook_both_land_when_nothing_races_them(tmp_path: Path, branch: str) -> None:
    """HFR-269 control — with nothing racing and nothing faulting, BOTH halves land.

    A combined operation that simply refused everything would satisfy the two tests
    below vacuously: "no hook was queued" and "the watch was not retired" are precisely
    what a ``_commit_cycle_result`` that always returned False, or an
    ``upsert_watch_with_queued_run`` that never committed, would produce. So each
    terminal branch needs its positive half too, and this is it -- the same real
    ``_run_watch`` over the same real ``agent_runs`` ledger, with NO teardown racing the
    write and NO injected outbox fault. The one transaction must COMMIT, carrying both
    of the things it makes atomic:

    * the branch's own result transition, read back from the definition (that branch's
      terminal exit code, disabled, retired, finished), and
    * exactly ONE outbox row, carrying that branch's OWN prompt text -- so a branch
      cannot be credited with a row some other branch wrote, and "exactly one" rules
      out the atomic write and the fallback ``enqueue`` both firing.

    Parametrized over every row of ``_HOOK_BRANCHES``, which is every hook-producing
    ``_commit_cycle_result`` call site in ``_run_watch``: success, per-cycle timeout,
    terminal failure, lifetime expiry, and missing cwd.
    """
    _store, service, watch, request_store, _session_id, calls, _engines = (
        _hook_branch_service_on_the_run_ledger(tmp_path, branch)
    )

    asyncio.run(service._run_watch(watch.id))

    _assert_branch_was_reached(branch, calls)

    stored = ManagedWatchStore().get_watch(watch.id)
    assert stored is not None, f"the {branch} branch's definition is gone"
    assert stored.last_exit_code == _HOOK_BRANCHES[branch]["expect_exit_code"], (
        f"the {branch} branch's result transition did not commit: the definition records "
        f"exit {stored.last_exit_code!r}, not {_HOOK_BRANCHES[branch]['expect_exit_code']!r}"
    )
    assert stored.enabled is False and stored.retired_at is not None, (
        f"the {branch} branch is terminal, but the definition is still enabled/unretired "
        f"(enabled={stored.enabled!r}, retired_at={stored.retired_at!r}): the stamp was "
        "refused or rolled back when nothing opposed it"
    )
    assert stored.last_finished_at is not None, (
        f"the {branch} branch's stamp committed without recording that the cycle finished"
    )

    pending = request_store.list_pending()
    assert len(pending) == 1, (
        f"the {branch} branch queued {len(pending)} hooks on the run ledger, expected "
        "exactly 1; with nothing racing it, the hook the branch authorises must be durable"
    )
    assert _HOOK_BRANCHES[branch]["expect_hook"] in (pending[0].prompt or ""), (
        f"the queued hook is not the {branch} branch's own: {pending[0].prompt!r}"
    )
    assert pending[0].task_id == watch.id, (
        f"the queued hook belongs to {pending[0].task_id!r}, not to the watch under test"
    )


@pytest.mark.parametrize("branch", list(_HOOK_BRANCHES))
def test_no_teardown_can_commit_between_the_result_stamp_and_its_hook(tmp_path: Path, branch: str) -> None:
    """HFR-269 — the stamp and the hook it authorises must be ONE transaction.

    THE PRODUCTION STORY. A watch bound to a Session finishes a cycle. In the same
    instant the user archives that Session: the archive runs
    ``reclaim_bound_definitions(mode='delete')``, which soft-deletes the watch, and the
    confirm dialog reports it.

    ORDERING WAS NOT ENOUGH. HFR-267 put the guarded stamp before the hook, so the
    guard now runs before the effect it authorises — but the two are separate
    transactions:

        self.store.mark_cycle_result(...)              # transaction 1, COMMITS
        self.request_store.enqueue_hook_send(...)       # transaction 2, COMMITS

    and the archive commits in the GAP between those commits. The stamp is accepted —
    it won its compare-and-set fairly, against the state that existed when it ran — and
    the hook is queued afterwards anyway: a durable request row the drain loop will
    deliver into a session that no longer exists, under a definition the database has
    soft-deleted, after the user was told the archive was done. Nothing is refused,
    because by the time the teardown lands there is nothing left to refuse.

    Driven through the REAL ``_run_watch`` over the REAL ``agent_runs`` ledger, with the
    real archive reclaim attempted from a second connection at the instant the outbox
    INSERT is about to run, and a zero busy timeout so "could it commit in the gap?" is
    answered by the database rather than by a sleep. Green requires that it could NOT:
    the write lock the stamp took is still held, so the teardown is serialised behind
    the single transaction and can only land before the stamp (where HFR-267's tests
    show it is refused) or after the hook is already durable.
    """
    store, service, watch, request_store, session_id, calls, engines = (
        _hook_branch_service_on_the_run_ledger(tmp_path, branch)
    )

    race = _try_archive_before_the_outbox_insert(
        engines, session_id, reason="the session was archived"
    )

    outcome: dict = {}
    try:
        asyncio.run(service._run_watch(watch.id))
    except Exception as exc:  # noqa: BLE001 - the split-transaction path can also fault here
        outcome["error"] = exc

    _assert_branch_was_reached(branch, calls)
    assert race["fired"] == 1, (
        f"the {branch} branch never wrote an outbox row, so this test proved nothing; "
        "the rendered SQL of the run INSERT drifted"
    )
    assert race["blocked"] is True, (
        f"the archive teardown COMMITTED between the {branch} branch's result stamp and "
        f"its hook ({race['summary']!r}): the two are separate transactions, so a "
        "reclaim lands in the gap and the hook is queued anyway — delivered into the "
        "session the archive removed, under a definition it soft-deleted"
    )
    assert not outcome, f"the atomic stamp+hook write faulted: {outcome.get('error')!r}"

    stored = ManagedWatchStore().get_watch(watch.id)
    assert stored is not None
    assert stored.enabled is False and stored.retired_at is not None, (
        f"the {branch} branch's terminal stamp did not land with its hook"
    )
    assert stored.last_exit_code == _HOOK_BRANCHES[branch]["expect_exit_code"], (
        f"the stamp recorded exit {stored.last_exit_code!r} for the {branch} branch"
    )
    pending = request_store.list_pending()
    assert len(pending) == 1, f"the {branch} branch queued {len(pending)} hooks, expected exactly 1"
    assert _HOOK_BRANCHES[branch]["expect_hook"] in (pending[0].prompt or ""), (
        f"the queued hook is not the {branch} branch's: {pending[0].prompt!r}"
    )


@pytest.mark.parametrize("branch", list(_HOOK_BRANCHES))
def test_a_failed_outbox_write_does_not_retire_the_watch_with_its_hook_lost(
    tmp_path: Path, branch: str
) -> None:
    """HFR-271: HFR-269, inverse half — the other thing two commits cost.

    Same scenario id as ``test_no_teardown_can_commit_between_the_result_stamp_and_its_hook``:
    one defect (the stamp and its outbox row are not one transaction) with two failure
    halves, tested separately because they fail for opposite reasons -- that one needs a
    teardown racing the gap, this one needs no race at all, just a fault on the second
    commit.

    HFR-267's ordering traded one failure for another. With the stamp committed first
    and the hook second, a fault on the outbox write — a full disk, a locked database,
    any error between the two commits — leaves the watch DURABLY DISABLED and retired
    while the hook that tells the user it finished is gone. A ``once`` watch is
    unrecoverable at that point: the definition says "finished", the user was never
    told, and nothing will run it again.

    One transaction removes the trade instead of reversing it: the outbox INSERT and the
    stamp roll back together, so the watch stays enabled and its completion is still
    owed. The fault is injected at the INSERT itself, on whichever connection the code
    under test writes the row through.
    """
    store, service, watch, request_store, _session_id, calls, engines = (
        _hook_branch_service_on_the_run_ledger(tmp_path, branch)
    )

    boom = _fail_the_outbox_insert(engines)

    outcome: dict = {}
    try:
        asyncio.run(service._run_watch(watch.id))
    except Exception as exc:  # noqa: BLE001 - the unfixed path lets the fault escape the loop
        outcome["error"] = exc

    _assert_branch_was_reached(branch, calls)
    assert boom["fired"] >= 1, (
        f"the {branch} branch never attempted an outbox INSERT, so no fault was injected"
    )
    assert request_store.list_pending() == [], "the failed INSERT queued a hook anyway"

    stored = ManagedWatchStore().get_watch(watch.id)
    assert stored is not None
    assert stored.enabled is True, (
        f"the {branch} branch retired the watch while the hook that reports its outcome "
        "was lost to the failed outbox write; a once watch is finished, silent and "
        "unrecoverable"
    )
    assert stored.retired_at is None and stored.last_finished_at is None, (
        "a terminal cycle result survived the failure of the hook it authorises"
    )

    # HFR-271 — AND THE OTHER HALF OF "IT ROLLED BACK". Everything above reads the
    # DURABLE row through a store this test just built. That is the half that was never
    # in doubt once the write became one transaction; the half that stayed wrong is the
    # SERVICE STILL RUNNING, whose ``ManagedWatch`` mirror ``mark_cycle_result`` mutated
    # BEFORE the write it then lost. A rollback is not proven by re-reading the row.
    live = service.store.get_watch(watch.id)
    assert live is not None, "the live store dropped the watch the database still has"
    assert (live.enabled, live.retired_at, live.last_finished_at) == (
        stored.enabled,
        stored.retired_at,
        stored.last_finished_at,
    ), (
        "the transaction rolled back and the LIVE store did not: it reports "
        f"enabled={live.enabled!r} retired_at={live.retired_at!r} "
        f"last_finished_at={live.last_finished_at!r} while the durable row says "
        f"enabled={stored.enabled!r} retired_at={stored.retired_at!r} "
        f"last_finished_at={stored.last_finished_at!r}. Every in-process reader "
        "(``reconcile_watches`` decides which watches to keep running from exactly "
        "this mirror) believes a watch retired that the database never retired"
    )
    assert live.last_exit_code == stored.last_exit_code, (
        f"the live mirror kept the rolled-back exit code {live.last_exit_code!r}"
    )


#: Any full-row write of a definition. Prefix-matched on both statement shapes so a
#: column change cannot silently stop the fault from being injected.
_DEFINITION_WRITES = ("UPDATE RUN_DEFINITIONS", "INSERT INTO RUN_DEFINITIONS")


def _fail_the_definition_write(engine) -> dict:
    """Make the ``run_definitions`` write itself fail, the way a real fault would."""
    from sqlalchemy import event

    state: dict = {"fired": 0}

    def _boom(conn, cursor, statement, parameters, context, executemany) -> None:  # noqa: ANN001
        normalized = " ".join(statement.split()).upper()
        if not normalized.startswith(_DEFINITION_WRITES):
            return
        state["fired"] += 1
        raise RuntimeError("definition write failed: disk I/O error")

    event.listens_for(engine, "before_cursor_execute")(_boom)
    return state


#: Every guarded writer that mutates the cached ``ManagedWatch`` before persisting it.
#: The value applies the writer and returns the fields it was supposed to change.
_MIRROR_WRITERS = {
    "mark_cycle_start": lambda store, watch_id: store.mark_cycle_start(watch_id),
    "mark_cycle_result": lambda store, watch_id: store.mark_cycle_result(
        watch_id, exit_code=0, error=None, disable=True
    ),
    "set_enabled": lambda store, watch_id: store.set_enabled(watch_id, False),
    "update_watch": lambda store, watch_id: store.update_watch(
        watch_id,
        **{**_WATCH_FIXTURE_PAYLOAD, "name": "renamed", "session_id": None},
    ),
}

#: Values a round trip through ``run_definitions`` returns unchanged, so a baseline
#: mismatch cannot be mistaken for a mirror the failed write left ahead.
_WATCH_FIXTURE_PAYLOAD = {
    "name": "original",
    "session_key": "slack::channel::C1",
    "command": ["true"],
    "shell_command": None,
    "prefix": None,
    "cwd": None,
    "mode": "once",
    "timeout_seconds": 1.0,
    "lifetime_timeout_seconds": 0.0,
    "retry_exit_codes": [75],
    "retry_delay_seconds": 30.0,
    "post_to": None,
    "deliver_key": None,
}


def test_a_hook_queued_after_a_rename_names_an_agent_that_still_exists(tmp_path: Path) -> None:
    """HFR-281: The watch half of SCT-030 -- the completion hook must be claimable.

    ``_hook_request`` composes the hook from ``watch.agent_name`` when the cycle ends,
    and the row becomes durable inside the stamp's transaction. A rename committed
    between those two moments rewrites the definition and every ACTIVE run, but not a
    row still being composed, so the hook was inserted naming an Agent the catalog no
    longer had -- and the user is never told the watch finished, which for a ``once``
    watch is the only report it ever produces.

    The mirror reload is forced because ``PRAGMA data_version`` is timing-dependent; a
    rename the mirror has NOT absorbed is refused by the compare-and-set instead, which
    needs no fix.
    """

    from core.vibe_agents import VibeAgentStore

    store = ManagedWatchStore()
    assert store._sqlite is not None, "the shared transaction this covers lives in SQLite"
    request_store = TaskExecutionStore()
    assert request_store._sqlite is not None

    agent_store = VibeAgentStore(paths.get_sqlite_state_path())
    try:
        # A user Agent: a built-in one cannot be renamed at all.
        agent_store.create(name="ops", backend="claude")
    finally:
        agent_store.close()

    watch = store.add_watch(**{**_WATCH_FIXTURE_PAYLOAD, "agent_name": "ops"})
    hook = request_store.build_hook_send(
        session_key=watch.session_key,
        prompt="the watch finished",
        agent_name=watch.agent_name,
        run_type="watch",
        definition_id=watch.id,
        source_kind="watch",
    )

    agent_store = VibeAgentStore(paths.get_sqlite_state_path())
    try:
        renamed = agent_store.rename("ops", "night-shift")
    finally:
        agent_store.close()
    store.load()
    assert store.get_watch(watch.id).agent_name == "night-shift"

    assert store.mark_cycle_result(
        watch.id,
        exit_code=0,
        error=None,
        event_detected=True,
        disable=True,
        queued_run=hook.to_dict(),
    ) is True

    rows = [row for row in request_store._sqlite.list_runs() if row["id"] == hook.id]
    assert len(rows) == 1, f"the hook did not become durable: {rows!r}"
    assert rows[0]["agent_name"] == "night-shift", (
        "the hook was queued against the pre-rename name, which resolves to no Agent "
        f"at claim time: {rows[0]['agent_name']!r}"
    )
    assert rows[0]["agent_id"] == renamed.id, (
        f"the hook was not pinned to the Agent's durable identity: {rows[0]['agent_id']!r}"
    )


@pytest.mark.parametrize("writer", list(_MIRROR_WRITERS))
def test_a_definition_write_that_raises_leaves_no_live_mirror_ahead_of_the_database(
    tmp_path: Path, writer: str
) -> None:
    """HFR-271 — when the write raises, the in-process mirror must roll back with it.

    THE RULE THIS TEST EXISTS FOR. ``ManagedWatchStore`` is a WRITE-THROUGH CACHE: every
    writer below edits the cached ``ManagedWatch`` first and hands the whole row to
    ``_write_watch`` second. ``_write_watch`` already reloads when the compare-and-set
    RETURNS ``False`` — and only then. When the write RAISES, the mirror keeps edits the
    database rolled back, and the process goes on serving them: ``reconcile_watches``
    picks which watches keep running out of exactly this dict, ``vibe watch list``
    renders it, and the next guarded write derives its ``expect`` snapshot from it
    (``_read_state``), so a stale mirror also sends the NEXT compare-and-set the wrong
    expectation.

    Nothing about that is specific to the outbox fault HFR-269 found; the outbox fault is
    just the first caller that made the raising path reachable in a test. So this is
    parametrized over EVERY guarded writer that goes through the shared choke point, and
    the fault is injected at the ``run_definitions`` statement itself — a disk error, a
    locked database, anything — rather than at one caller's second write.
    """
    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test is about the guarded SQLite path"
    watch = store.add_watch(**_WATCH_FIXTURE_PAYLOAD)

    durable_before = ManagedWatchStore().get_watch(watch.id)
    live_before = store.get_watch(watch.id)
    assert durable_before is not None and live_before is not None
    assert live_before.to_dict() == durable_before.to_dict(), (
        "the fixture starts with the mirror already disagreeing with the row, so the "
        "assertion below would pass or fail for the wrong reason"
    )
    boom = _fail_the_definition_write(store._sqlite.engine)

    with pytest.raises(Exception):  # noqa: B017 - the fault type is the injected one's
        _MIRROR_WRITERS[writer](store, watch.id)

    assert boom["fired"] >= 1, (
        f"{writer} never wrote a run_definitions row, so no fault was injected and this "
        "test proves nothing"
    )

    durable_after = ManagedWatchStore().get_watch(watch.id)
    assert durable_after is not None
    assert durable_after.to_dict() == durable_before.to_dict(), (
        f"{writer} committed something despite the injected fault; this test can no "
        "longer tell a rolled-back mirror from a written one"
    )

    live = store.get_watch(watch.id)
    assert live is not None, f"{writer} dropped the watch from the live store"
    assert live.to_dict() == durable_after.to_dict(), (
        f"{writer} left the live mirror ahead of the database. The transaction rolled "
        "back; the cached ManagedWatch kept the edit. Differing fields: "
        + repr(
            {
                key: (value, durable_after.to_dict().get(key))
                for key, value in live.to_dict().items()
                if durable_after.to_dict().get(key) != value
            }
        )
    )


def test_a_failed_create_leaves_no_phantom_watch_in_the_live_store() -> None:
    """HFR-275 — the create entry point must roll its mirror back as well.

    ``upsert_watch`` is the one writer that can put an id in the mirror the database has
    NEVER seen, and it inserted into ``self._watches`` before writing. It is also the
    worst place to leave a mirror ahead: the caller is told the watch could not be
    created, while ``reconcile_watches`` -- which picks what to run out of exactly this
    dict -- STARTS it, spawning its command and posting its output on every tick, with no
    durable row to stop it and nothing that will ever reload it away.
    """
    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test is about the guarded SQLite path"
    before = {watch.id for watch in store.list_watches()}
    boom = _fail_the_definition_write(store._sqlite.engine)

    with pytest.raises(Exception):  # noqa: B017 - the fault type is the injected one's
        store.add_watch(**_WATCH_FIXTURE_PAYLOAD)

    assert boom["fired"] >= 1, "no run_definitions write was attempted, so nothing failed"
    phantom = {watch.id for watch in store.list_watches()} - before
    assert not phantom, (
        f"the failed create left {sorted(phantom)} in the live store: a watch the "
        "database never accepted, that reconcile_watches will start running anyway"
    )
    assert not {watch.id for watch in ManagedWatchStore().list_watches()} - before, (
        "the create committed despite the injected fault, so this test proves nothing"
    )


def test_a_failed_delete_does_not_stop_a_watch_the_database_still_has() -> None:
    """HFR-275 — the delete entry point, the same class in the safer direction.

    ``remove_watch`` drops the entry before the soft delete. The failure direction is the
    conservative one (absent reads as "gone" and stops the watch), but it is silent and
    it does NOT heal: the row is still there and UNCHANGED, so ``maybe_reload`` sees no
    external write, and the watch the user was told could not be deleted simply stops
    until the process restarts.
    """
    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test is about the guarded SQLite path"
    watch = store.add_watch(**_WATCH_FIXTURE_PAYLOAD)
    boom = _fail_the_definition_write(store._sqlite.engine)

    with pytest.raises(Exception):  # noqa: B017 - the fault type is the injected one's
        store.remove_watch(watch.id)

    assert boom["fired"] >= 1, "no run_definitions write was attempted, so nothing failed"
    durable = ManagedWatchStore().get_watch(watch.id)
    assert durable is not None, (
        "the delete committed despite the injected fault, so this test proves nothing"
    )
    live = store.get_watch(watch.id)
    assert live is not None, (
        f"the failed delete dropped watch {watch.id} from the live store while the "
        "database still has it: it stops running, silently, until the process restarts"
    )
    assert live.to_dict() == durable.to_dict()


#: The list read ``ManagedWatchStore.load`` issues -- the only ``run_definitions``
#: SELECT that filters by ``definition_type`` -- so the guard's own
#: ``SELECT id ... LIMIT 1`` still runs and the write is genuinely ATTEMPTED.
_DEFINITION_LIST_READ_MARKERS = ("FROM RUN_DEFINITIONS", "DEFINITION_TYPE = ?")


def _fail_the_definition_write_and_the_reload(engine) -> dict:
    """A transient fault that takes out the guarded write AND the recovery read.

    The task store's twin helper. Flip ``state["live"]`` to ``False`` to end the fault
    WITHOUT committing anything: a commit would bump ``PRAGMA data_version`` and heal the
    mirror for a reason that has nothing to do with the fix.
    """
    from sqlalchemy import event

    state: dict = {"writes": 0, "reads": 0, "live": True}

    def _boom(conn, cursor, statement, parameters, context, executemany) -> None:  # noqa: ANN001
        if not state["live"]:
            return
        normalized = " ".join(statement.split()).upper()
        if normalized.startswith(("UPDATE RUN_DEFINITIONS", "INSERT INTO RUN_DEFINITIONS")):
            state["writes"] += 1
            raise RuntimeError("definition write failed: disk I/O error")
        if all(marker in normalized for marker in _DEFINITION_LIST_READ_MARKERS):
            state["reads"] += 1
            raise RuntimeError("definition read failed: disk I/O error")

    event.listens_for(engine, "before_cursor_execute")(_boom)
    return state


def test_a_dropped_watch_mirror_recovers_with_no_unrelated_commit_to_wake_it() -> None:
    """HFR-277 — the consuming end of HFR-271's own recovery path.

    THE DEFECT, and it is one WE introduced. ``_reload_after_lost_write`` drops the
    cached ``ManagedWatch`` when the guarded write fails and the immediate recovery
    ``load`` fails with it -- deliberately, because an absent watch reads as "gone" to
    ``reconcile_watches`` and stops it, where a stale one keeps running against state the
    database never had -- and promises ``maybe_reload`` restores it "as soon as the
    database is reachable again". It did not. ``maybe_reload`` asked only
    ``SqliteInvalidationProbe`` (``PRAGMA data_version``), which moves when another
    connection COMMITS; the failed write ROLLED BACK, so it never moved. Every later
    supervisor tick answered "nothing changed" and the watch stayed durably enabled in
    SQLite and invisible in-process until the service restarted.

    THE CLAUSE THAT IS THE TEST: nothing commits between the failure and the recovery.
    A witness probe on its own connection asserts data_version is unchanged at that
    instant, so a mirror that comes back did so because the store remembered it must
    reload -- not because an unrelated writer woke it up.
    """
    from storage.db import SqliteInvalidationProbe, create_sqlite_engine

    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test is about the guarded SQLite path"
    watch = store.add_watch(**_WATCH_FIXTURE_PAYLOAD)
    durable_before = ManagedWatchStore().get_watch(watch.id)
    assert durable_before is not None and durable_before.enabled, (
        "the fixture must start from a definition the database has, and has enabled"
    )

    # Settle the store's own probe on the fixture's commits first: a pending
    # data_version bump would reload the mirror below for the wrong reason.
    store.maybe_reload()
    assert store.maybe_reload() is False, "the store's probe is not settled"
    witness_engine = create_sqlite_engine(store._sqlite.db_path)
    witness = SqliteInvalidationProbe(witness_engine)
    witness.has_external_write()
    assert witness.has_external_write() is False, "the witness probe is not settled"

    fault = _fail_the_definition_write_and_the_reload(store._sqlite.engine)
    try:
        with pytest.raises(Exception):  # noqa: B017 - the fault type is the injected one's
            store.mark_cycle_start(watch.id)

        assert fault["writes"] >= 1, "no run_definitions write was attempted"
        assert fault["reads"] >= 1, (
            "the recovery reload was never attempted, so the entry was not dropped for "
            "the reason this test is about"
        )
        assert store.get_watch(watch.id) is None, (
            "the failed write did not drop the mirror entry, so there is nothing for "
            "maybe_reload to recover and this test proves nothing"
        )
        assert [item.id for item in store.list_watches()] == [], (
            "the dropped entry is still listed; the precondition is a mirror that has "
            "LOST the definition"
        )

        # The fault clears the way a transient one does: nothing is written.
        fault["live"] = False
        assert witness.has_external_write() is False, (
            "something COMMITTED between the failed write and the reload below. A "
            "data_version bump heals the mirror on its own, so this test would pass "
            "without the fix"
        )

        assert store.maybe_reload() is True, (
            "maybe_reload reported 'nothing changed' for a mirror the store itself knows "
            "is incomplete. data_version cannot see a rolled-back write, so the dropped "
            "watch stays invisible to reconcile_watches until the process restarts"
        )
        live = store.get_watch(watch.id)
        assert live is not None, (
            f"watch {watch.id} is still missing from the live store after a reload; it is "
            "enabled in SQLite and will never be reconciled again"
        )
        assert live.to_dict() == durable_before.to_dict(), (
            "the recovered entry does not match the durable row. Differing fields: "
            + repr(
                {
                    key: (value, durable_before.to_dict().get(key))
                    for key, value in live.to_dict().items()
                    if durable_before.to_dict().get(key) != value
                }
            )
        )
        assert [item.id for item in store.list_watches()] == [watch.id]
        assert store.maybe_reload() is False, (
            "the store keeps reloading unconditionally; the flag must be cleared by the "
            "reload that repaired the mirror"
        )
    finally:
        witness.close()
        witness_engine.dispose()

    durable_after = ManagedWatchStore().get_watch(watch.id)
    assert durable_after is not None and durable_after.to_dict() == durable_before.to_dict(), (
        "the durable row changed, so the failed write committed something and the "
        "recovery above was reading a different definition than the one that was dropped"
    )


def test_quiet_cycle_marker_must_be_a_line_of_its_own() -> None:
    """The marker is a protocol line, not a substring.

    A waiter that prints the marker's name while explaining the protocol — a usage
    message, a log line quoting it — would otherwise have its exit 64 read as a
    quiet cycle, and its failure notice would never reach the user.
    """
    from core.watches import _is_quiet_cycle

    def _result(text: str) -> _CycleResult:
        return _CycleResult(exit_code=NO_EVENT_EXIT_CODE, stdout="", stderr=text, timed_out=False)

    assert _is_quiet_cycle(_result(f"nothing new\n{NO_EVENT_MARKER}\n"))
    # Indentation is still a line of its own; the surrounding prose is not.
    assert _is_quiet_cycle(_result(f"  {NO_EVENT_MARKER}  "))
    assert not _is_quiet_cycle(_result(f"usage: exit 64 with '{NO_EVENT_MARKER}' to end quietly"))
    assert not _is_quiet_cycle(_result(f"{NO_EVENT_MARKER} is the marker"))


def test_quiet_cycle_summary_keeps_the_end_of_a_long_waiter_report() -> None:
    """The verdict is at the end of a waiter's report, so truncate from the front.

    A green CI waiter prints the run table first and its conclusion last. Squashing
    from the front kept the table and dropped the one line worth logging.
    """
    from core.watches import NO_EVENT_SUMMARY_LOG_LIMIT, _squash_tail

    short = "All watched workflows succeeded: lint, build"
    assert _squash_tail(f"  {short}\n", limit=NO_EVENT_SUMMARY_LOG_LIMIT) == short

    long_report = ("workflow line\n" * 500) + short
    squashed = _squash_tail(long_report, limit=NO_EVENT_SUMMARY_LOG_LIMIT)
    assert len(squashed) <= NO_EVENT_SUMMARY_LOG_LIMIT
    assert squashed.endswith(short)
    assert squashed.startswith("…")


def test_a_cycle_is_told_when_this_watch_last_had_a_report_delivered(
    tmp_path: Path,
    monkeypatch,
) -> None:
    """HFR-469: a waiter learns whether its earlier report was delivered.

    A waiter cannot observe its own delivery: its stdout reaches the supervisor only
    after the process exits, so a waiter that wants to advance its own cursors past a
    reported event has to be told. What it is told is a count of delivered reports,
    bumped by ``_commit_cycle_result`` in the same transaction as the follow-up hook --
    so a waiter that staged cursors alongside the value it saw can compare, and a
    CHANGED value means its report was queued.

    A durable count rather than a one-shot flag handed to the next cycle: it is still
    correct after a restart, and for a ``once`` watch, whose one report is followed by
    no cycle at all until the user resumes it -- where a lost flag would replay an
    event that was delivered long ago. And it must NOT move on a quiet cycle, which
    queued nothing: a waiter would then promote cursors covering a report that was
    never delivered and skip the activity for good.
    """
    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test needs the SQLite-backed store"
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    session_id = _bare_watch_session_row(workdir=tmp_path, anchor="slack_C_ack")
    watch = store.add_watch(
        name="Watch PR",
        session_key="",
        session_id=session_id,
        session_policy="existing",
        command=[sys.executable, "-c", "print('event')"],
        shell_command=None,
        prefix="PR activity.",
        cwd=None,
        mode="forever",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
        metadata={"origin": "cli"},
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    service._running = True
    service._requires_service_lease = False

    # One event, one quiet cycle, then stop: the three states the flag has to
    # distinguish, in the order a forever watch meets them.
    scripted = [
        _CycleResult(exit_code=0, stdout="review_comment #501", stderr="", timed_out=False),
        _CycleResult(exit_code=NO_EVENT_EXIT_CODE, stdout="", stderr=NO_EVENT_MARKER, timed_out=False),
        _CycleResult(exit_code=NO_EVENT_EXIT_CODE, stdout="", stderr=NO_EVENT_MARKER, timed_out=False),
    ]
    told: list[str] = []

    async def _spy_cycle(watch_arg, *, timeout_seconds):  # noqa: ANN001
        told.append(_cycle_env(watch_arg)[LAST_DELIVERY_ENV])
        if len(told) >= len(scripted):
            service._running = False
        return scripted[min(len(told) - 1, len(scripted) - 1)]

    service._run_cycle = _spy_cycle  # type: ignore[method-assign]
    monkeypatch.setattr(watches_module, "WATCH_MIN_REARM_SECONDS", 0)
    monkeypatch.setattr(watches_module, "WATCH_FOLLOW_UP_POLL_SECONDS", 0.001)

    async def _run() -> None:
        task = asyncio.create_task(service._run_watch(watch.id))
        while not task.done():
            for request in request_store.list_pending():
                claimed = request_store.claim(request.id)
                if claimed is not None:
                    request_store.complete(claimed, ok=True)
            await asyncio.sleep(0.001)
        await task

    asyncio.run(_run())

    assert told[0] == "", "nothing has been reported yet"
    assert told[1], "the event was delivered, so the stamp has to have moved"
    assert told[2] == told[1], "a quiet cycle queued no hook; the stamp must not move"


def test_resuming_a_once_watch_keeps_what_it_already_delivered(tmp_path: Path) -> None:
    """A resume clears the cycle history and NOT the delivery acknowledgement.

    A ``once`` watch reports, retires, and is resumed by the user days later. That
    resume deliberately clears ``last_event_at`` -- the resumed watch begins a distinct
    cycle and its row must not claim an event it has not seen. But the waiter's staged
    cursors are still on disk beside the count they were staged against, and if that
    count resets too, the first cycle after the resume concludes its report never left
    and reports the same activity all over again.
    """
    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test needs the SQLite-backed store"
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    session_id = _bare_watch_session_row(workdir=tmp_path, anchor="slack_C_resume_ack")
    watch = store.add_watch(
        name="Watch PR once",
        session_key="",
        session_id=session_id,
        session_policy="existing",
        command=[sys.executable, "-c", "print('event')"],
        shell_command=None,
        prefix="PR activity.",
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
        metadata={"origin": "cli"},
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )
    service._running = True
    service._requires_service_lease = False

    async def _one_event(watch_arg, *, timeout_seconds):  # noqa: ANN001
        return _CycleResult(exit_code=0, stdout="review_comment #501", stderr="", timed_out=False)

    service._run_cycle = _one_event  # type: ignore[method-assign]
    asyncio.run(service._run_watch(watch.id))

    reported = store.get_watch(watch.id)
    assert reported is not None and not reported.enabled, "a once watch retires on its report"
    delivered = _cycle_env(reported)[LAST_DELIVERY_ENV]
    assert delivered == "1"

    resumed = store.set_enabled(watch.id, True)
    assert resumed.last_event_at is None, "the resume clears the cycle history it owns"
    assert _cycle_env(resumed)[LAST_DELIVERY_ENV] == delivered
    assert resumed.metadata["origin"] == "cli", "the ack rides along with the rest of the metadata"


def test_the_cycle_environment_carries_the_watch_and_its_delivery_count() -> None:
    """Both facts reach the waiter as environment variables, empty for never.

    Metadata a writer never touched, or left as something other than a positive count,
    reads as "nothing delivered yet" -- the direction that reports a staged event twice
    rather than advancing past one that never left.
    """
    assert _cycle_env(SimpleNamespace(id="wat_9", metadata={})) == {
        WATCH_ID_ENV: "wat_9",
        LAST_DELIVERY_ENV: "",
    }
    assert _cycle_env(SimpleNamespace(id="wat_9", metadata={DELIVERY_ACK_METADATA_KEY: 3})) == {
        WATCH_ID_ENV: "wat_9",
        LAST_DELIVERY_ENV: "3",
    }
    for broken in ("3", True, 0, -1, None, {"count": 3}):
        assert _cycle_env(SimpleNamespace(id="wat_9", metadata={DELIVERY_ACK_METADATA_KEY: broken})) == {
            WATCH_ID_ENV: "wat_9",
            LAST_DELIVERY_ENV: "",
        }


def test_managed_watch_service_names_the_watch_in_the_cycle_environment(tmp_path: Path) -> None:
    """A waiter can tell which watch it is a cycle of.

    Waiters that keep per-watch state on disk own that state by this id, so two
    identically configured watches cannot silently share one state file.
    """
    store = ManagedWatchStore(tmp_path / "watches.json")
    request_store = TaskExecutionStore(tmp_path / "task_requests")
    runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
    echoed = tmp_path / "watch-id.txt"
    watch = store.add_watch(
        name="Identity echo waiter",
        session_key="slack::channel::C123",
        command=[
            sys.executable,
            "-c",
            (
                "import os, pathlib; "
                f"pathlib.Path({str(echoed)!r}).write_text(os.environ.get('AVIBE_WATCH_ID', ''))"
            ),
        ],
        shell_command=None,
        prefix="Something happened.",
        cwd=None,
        mode="once",
        timeout_seconds=5,
        lifetime_timeout_seconds=0,
        retry_exit_codes=[75],
        retry_delay_seconds=0.01,
        post_to=None,
        deliver_key=None,
    )
    service = ManagedWatchService(
        controller=SimpleNamespace(),
        store=store,
        request_store=request_store,
        runtime_store=runtime_store,
    )

    async def _run() -> None:
        service.start()
        for _ in range(100):
            if echoed.exists() and echoed.read_text(encoding="utf-8"):
                break
            await asyncio.sleep(0.02)
        await service.stop()

    asyncio.run(_run())

    assert echoed.read_text(encoding="utf-8") == watch.id


def test_a_watch_committed_before_a_failed_reload_is_not_lost_with_the_probe() -> None:
    """The commit a failed reload consumed must still reach the mirror.

    ``PRAGMA data_version`` reports another connection's commit once. When the
    reload it triggers fails -- as every read does while the disk is full -- a
    later probe no longer sees that commit, so a watch created during the outage
    stayed enabled in SQLite and invisible to reconcile until an unrelated commit
    or a restart. Nothing commits between the failure and the recovery below, so
    the mirror can only heal because the store remembered it must reload.
    """
    from storage.db import SqliteInvalidationProbe, create_sqlite_engine

    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test is about the SQLite probe"
    store.maybe_reload()
    assert store.maybe_reload() is False, "the store's probe is not settled"

    created = ManagedWatchStore().add_watch(**_WATCH_FIXTURE_PAYLOAD)
    witness_engine = create_sqlite_engine(store._sqlite.db_path)
    witness = SqliteInvalidationProbe(witness_engine)
    witness.has_external_write()
    assert witness.has_external_write() is False, "the witness probe is not settled"

    fault = _fail_the_definition_write_and_the_reload(store._sqlite.engine)
    try:
        with pytest.raises(Exception):  # noqa: B017 - the fault type is the injected one's
            store.maybe_reload()
        assert fault["reads"] >= 1, "the probe did not report the commit, so no reload failed"
        assert store.get_watch(created.id) is None, "the mirror already holds the new watch"

        fault["live"] = False
        assert witness.has_external_write() is False, (
            "something COMMITTED after the failed reload; a data_version bump heals "
            "the mirror on its own, so this test would pass without the fix"
        )
        assert store.maybe_reload() is True, (
            "the failed reload consumed the only change signal, so the watch created "
            "during the outage stays invisible to reconcile"
        )
        assert [item.id for item in store.list_watches()] == [created.id]
        assert store.maybe_reload() is False, "the repairing reload must clear the flag"
    finally:
        witness.close()
        witness_engine.dispose()


def test_a_watch_read_never_waits_for_a_writer_holding_sqlite_and_never_sees_its_unwritten_edit(monkeypatch) -> None:
    """#2705: watch writers run on worker threads and hold the mirror lock across their SQLite write, which waits
    while another process holds SQLite's writer. The controller loop reads the mirror on every watch tick
    (``get_watch``, ``list_watches``), so a read takes no lock: it answers at once, from the published mirror, which
    a writer replaces only after its write lands."""

    import sqlite3

    store = ManagedWatchStore()
    assert store._sqlite is not None, "this test is about a writer waiting on SQLite"
    watch = store.add_watch(**_WATCH_FIXTURE_PAYLOAD)
    writer_writing = threading.Event()
    original_upsert = store._sqlite.upsert_watch

    def upsert(*args, **kwargs):
        writer_writing.set()
        return original_upsert(*args, **kwargs)

    monkeypatch.setattr(store._sqlite, "upsert_watch", upsert)
    other_writer_holds, release_other_writer = threading.Event(), threading.Event()

    def another_process_holds_the_writer() -> None:
        holder = sqlite3.connect(str(store._sqlite.db_path), timeout=10)
        try:
            holder.execute("BEGIN IMMEDIATE")
            other_writer_holds.set()
            release_other_writer.wait(1.0)
            holder.rollback()
        finally:
            holder.close()

    holder = threading.Thread(target=another_process_holds_the_writer)
    holder.start()
    assert other_writer_holds.wait(5)
    worker = threading.Thread(target=store.set_enabled, args=(watch.id, False))
    worker.start()
    try:
        assert writer_writing.wait(5)
        time.sleep(0.1)  # the worker's write now waits for SQLite's writer, holding the mirror lock
        started = time.monotonic()
        current = store.get_watch(watch.id)
        # Read now: a landed write later updates this same entry, by design.
        read_enabled = (current.enabled if current is not None else None, [item.enabled for item in store.list_watches()])
        waited = time.monotonic() - started
    finally:
        release_other_writer.set()
        holder.join(5)
        worker.join(5)

    assert waited < 0.05, f"a watch read waited {waited:.2f}s for a writer holding SQLite"
    assert read_enabled == (True, [True]), "a reader saw the writer's unwritten edit"
    assert store.get_watch(watch.id).enabled is False, "the landed write was never published"