Skip to content

Autonomous research system for measurable, computer-executable research.

Python
108 KB
Raw Blame
"""Tests for ``praxist stop`` — CLI lifecycle Phases 3 + 4."""

from __future__ import annotations

import io
import json
import os
import tempfile
import unittest
from contextlib import contextmanager, redirect_stderr, redirect_stdout
from pathlib import Path
from unittest.mock import patch


def _entry_kwargs(**overrides: object) -> dict[str, object]:
    base = {
        "schema_version": 1,
        "run_id": "run_stop_demo",
        "pid": 5000,
        "parent_pid": 1,
        "run_dir": "/tmp/stop_demo",
        "log_file": "/tmp/stop_demo/log",
        "task_path": "/tmp/task",
        "model": "claude-opus-4-7",
        "model_provider_ref": "model_provider:anthropic_messages",
        "runtime_ref": "agent_runtime:claude_sdk",
        "command": ("python", "-m", "praxist.run", "run", "--task-path", "/t"),
        "command_prefix": "python -m praxist.run",
        "started_at": "2026-05-19T09:00:00+00:00",
    }
    base.update(overrides)
    return base


def _write_entry(**overrides: object) -> str:
    from praxist.cli import registry

    if "run_dir" not in overrides:
        state_dir = Path(os.environ["PRAXIST_STATE_DIR"])
        run_id = str(overrides.get("run_id") or "run_stop_demo")
        run_dir = state_dir / "run_dirs" / run_id
        overrides["run_dir"] = str(run_dir)
        overrides.setdefault("log_file", str(run_dir / "log"))
    entry = registry.RegistryEntry(**_entry_kwargs(**overrides))
    Path(entry.run_dir).mkdir(parents=True, exist_ok=True)
    registry.write_entry(entry)
    return entry.run_id


class _StateDirMixin:
    """Set up an isolated PRAXIST_STATE_DIR for each test."""

    def setUp(self) -> None:  # type: ignore[override]
        self.tmp = tempfile.TemporaryDirectory()
        self.addCleanup(self.tmp.cleanup)
        self._env_patch = patch.dict(os.environ, {"PRAXIST_STATE_DIR": self.tmp.name}, clear=False)
        self._env_patch.start()
        self.addCleanup(self._env_patch.stop)
        self._token_patch = patch(
            "praxist.cli.stop.process_start_token",
            side_effect=lambda pid: f"proc:{pid}",
        )
        self._token_patch.start()
        self.addCleanup(self._token_patch.stop)


class StopRunTest(_StateDirMixin, unittest.TestCase):
    """Selective ``praxist stop <run_id>`` paths."""

    def test_live_pid_signalled_then_marked_stopped(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry()
        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
            5001: (5000, "00:09", "python /t/variants/v0/train.py"),
        }
        killed: list[tuple[int, int]] = []
        alive_pids = {5000, 5001}

        def fake_kill(pid: int, sig: int) -> None:
            killed.append((pid, sig))
            # SIGTERM clears the alive flag on next check (simulate fast exit).
            if pid in alive_pids:
                alive_pids.discard(pid)

        def fake_alive(pid: int) -> bool:
            return pid in alive_pids

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", side_effect=fake_alive),
            patch("praxist.cli.stop.os.kill", side_effect=fake_kill),
            patch("praxist.cli.stop.time.sleep"),
            patch(
                "praxist.cli.stop._utc_now_iso",
                return_value="2026-05-19T09:30:00+00:00",
            ),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.05)

        # Root + descendant signalled.
        self.assertEqual(outcome.matched_pids, [5000])
        self.assertEqual(outcome.descendant_pids, [5001])
        # SIGTERM sent to both.
        sigterms = [p for p, s in killed if s == 15]
        self.assertEqual(sorted(sigterms), [5000, 5001])
        # No SIGKILL needed because both exited.
        self.assertEqual(outcome.killed_pids, [])
        self.assertEqual(outcome.remaining_pids, [])
        # Registry marked stopped.
        on_disk = registry.read_entry(run_id)
        self.assertEqual(on_disk.state, registry.STATE_STOPPED)
        self.assertEqual(on_disk.stopped_at, "2026-05-19T09:30:00+00:00")

    def test_stop_writes_shutdown_fence_before_process_discovery(self) -> None:
        from praxist.cli import stop

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            run_dir.mkdir()
            run_id = _write_entry(run_dir=str(run_dir))

            def snapshot_after_fence():
                self.assertTrue((run_dir / "ORCHESTRATOR_SHUTDOWN").is_file())
                return (
                    {5000: (1, "00:10", "python -m praxist.run run --task-path /t")},
                    {5000: "proc:5000"},
                )

            with (
                patch(
                    "praxist.cli.stop._read_identity_bound_ps_snapshot",
                    side_effect=snapshot_after_fence,
                ),
                patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
                patch("praxist.cli.stop.pid_is_alive", return_value=False),
                patch("praxist.cli.stop.os.kill"),
                patch("praxist.cli.stop._drain_late_run_processes", return_value={}),
            ):
                outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertEqual(outcome.remaining_pids, [])

    def test_stop_refuses_to_signal_when_admission_cannot_close(self) -> None:
        from praxist.cli import registry, stop

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            run_dir.mkdir()
            run_id = _write_entry(run_dir=str(run_dir))
            with (
                patch(
                    "praxist.cli.stop._write_run_shutdown_fence",
                    return_value=False,
                ),
                patch("praxist.cli.stop._read_identity_bound_ps_snapshot") as snapshot,
                patch("praxist.cli.stop.os.kill") as kill,
                self.assertRaisesRegex(stop.StopError, "could not close experiment admission"),
            ):
                stop.stop_run(run_id=run_id, grace_seconds=0.0)

        snapshot.assert_not_called()
        kill.assert_not_called()
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_scheduler_freeze_allows_stop_when_shutdown_file_is_unwritable(self) -> None:
        from praxist.cli import registry, stop

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            endpoint = run_dir / "resource_scheduler" / "endpoint.json"
            endpoint.parent.mkdir(parents=True)
            endpoint.write_text("{}\n", encoding="utf-8")
            run_id = _write_entry(run_dir=str(run_dir))
            scheduler_client = (
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler_client.freeze_all_for_run"
            )
            with (
                patch(
                    "praxist.cli.stop._write_run_shutdown_fence",
                    return_value=False,
                ),
                patch(scheduler_client, return_value=True),
                patch(
                    "praxist.cli.stop._read_identity_bound_ps_snapshot",
                    return_value=({}, {}),
                ),
                patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
                patch("praxist.cli.stop._drain_late_run_processes", return_value={}),
                patch("praxist.cli.stop.pid_is_alive", return_value=False),
            ):
                outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertEqual(outcome.remaining_pids, [])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)

    def test_stop_drains_process_forked_after_initial_snapshot(self) -> None:
        from praxist.cli import registry, stop

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            run_dir.mkdir()
            run_id = _write_entry(run_dir=str(run_dir))
            alive = {5000}
            signals: list[tuple[int, int]] = []
            late_forked = False

            def fake_kill(pid: int, sig: int) -> None:
                nonlocal late_forked
                signals.append((pid, sig))
                alive.discard(pid)
                if pid == 5000 and sig == 15 and not late_forked:
                    alive.add(6001)
                    late_forked = True

            def discover_late(_run_dir: Path):
                return (
                    ({6001: "proc:6001"} if 6001 in alive else {}),
                    {},
                )

            with (
                patch(
                    "praxist.cli.stop._read_identity_bound_ps_snapshot",
                    return_value=(
                        {5000: (1, "00:10", "python -m praxist.run run --task-path /t")},
                        {5000: "proc:5000"},
                    ),
                ),
                patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
                patch(
                    "praxist.cli.stop._discover_run_owned_processes",
                    side_effect=discover_late,
                ),
                patch("praxist.cli.stop.pid_is_alive", side_effect=lambda pid: pid in alive),
                patch("praxist.cli.stop.os.kill", side_effect=fake_kill),
                patch("praxist.cli.stop.time.sleep"),
            ):
                outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertIn((6001, 15), signals)
        self.assertIn(6001, outcome.descendant_pids)
        self.assertEqual(outcome.remaining_pids, [])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)

    def test_late_drain_escalates_persistent_owned_process(self) -> None:
        from praxist.cli import stop

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td)
            outcome = stop.StopOutcome(run_id="run")
            with (
                patch.object(stop, "_LATE_DRAIN_MAX_SCANS", 3),
                patch.object(stop, "_LATE_DRAIN_SCAN_SECONDS", 0),
                patch.object(stop, "_LATE_DRAIN_TERM_SCANS", 2),
                patch.object(
                    stop,
                    "_discover_run_owned_processes",
                    return_value=({41: "proc:41"}, {}),
                ),
                patch.object(
                    stop,
                    "_live_process_instances",
                    side_effect=[{41}, {41}, {41}, set(), set(), set(), set()],
                ),
                patch.object(stop, "_signal_process_groups") as signal_groups,
                patch.object(stop, "_signal_set") as signal_set,
                patch.object(stop.time, "sleep"),
            ):
                remaining = stop._drain_late_run_processes({"run": run_dir}, outcome)

        self.assertEqual(remaining, {})
        self.assertTrue(
            any(call.args[1] == stop.signal.SIGKILL for call in signal_groups.call_args_list)
        )
        self.assertTrue(
            any(call.args[1] == stop.signal.SIGKILL for call in signal_set.call_args_list)
        )

    def test_late_drain_final_verification_reports_new_survivor(self) -> None:
        from praxist.cli import stop

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td)
            outcome = stop.StopOutcome(run_id="run")
            with (
                patch.object(stop, "_LATE_DRAIN_MAX_SCANS", 1),
                patch.object(stop, "_LATE_DRAIN_SCAN_SECONDS", 0),
                patch.object(
                    stop,
                    "_discover_run_owned_processes",
                    return_value=({42: "proc:42"}, {}),
                ),
                patch.object(
                    stop,
                    "_live_process_instances",
                    side_effect=[{42}, {42}, set(), set(), {42}],
                ),
                patch.object(stop, "_signal_process_groups"),
                patch.object(stop, "_signal_set"),
                patch.object(stop, "pid_is_alive", return_value=False),
                patch.object(stop.time, "sleep"),
            ):
                remaining = stop._drain_late_run_processes({"run": run_dir}, outcome)

        self.assertEqual(remaining, {"run": {42}})
        self.assertEqual(outcome.remaining_pids, [42])

    def test_shutdown_fence_and_process_identity_boundary_fail_closed(self) -> None:
        from praxist.cli import stop

        outcome = stop.StopOutcome(run_id="run")
        with (
            patch.object(Path, "exists", return_value=False),
            patch.object(Path, "write_text", side_effect=OSError("read-only")),
        ):
            self.assertFalse(stop._write_run_shutdown_fence(Path("/run"), outcome, source="test"))
        self.assertIn("read-only", outcome.warnings[0])

        raw = b"PRAXIST_RUN_DIR=bad\0AUTO_RESEARCH_RUN_DIR=/tmp/run\0"
        with (
            patch.object(Path, "read_bytes", return_value=raw),
            patch.object(stop.os, "fsdecode", side_effect=[UnicodeError(), "/tmp/run"]),
        ):
            self.assertEqual(
                stop._run_dir_from_process_environment(123),
                Path("/tmp/run").resolve(),
            )

    def test_pidfd_helpers_use_exact_process_handle_when_available(self) -> None:
        from praxist.cli import stop

        with patch.object(stop.os, "pidfd_open", return_value=17, create=True):
            self.assertEqual(stop._open_process_handle(41), 17)
        with patch.object(stop.os, "pidfd_open", return_value=object(), create=True):
            self.assertIsNone(stop._open_process_handle(41))
        with patch.object(stop.os, "pidfd_open", side_effect=OSError("gone"), create=True):
            self.assertIsNone(stop._open_process_handle(41))

        with patch.object(stop.signal, "pidfd_send_signal", create=True) as sender:
            stop._signal_process_handle(17, stop.signal.SIGTERM)
        sender.assert_called_once_with(17, stop.signal.SIGTERM, None, 0)

    def test_pidfd_helpers_use_libc_fallback_when_stdlib_api_is_missing(self) -> None:
        import ctypes

        from praxist.cli import stop

        class FakeFunction:
            def __init__(self, result: int) -> None:
                self.result = result
                self.argtypes = None
                self.restype = None

            def __call__(self, *_args: object) -> int:
                return self.result

        class FakeLibc:
            def __init__(self, *, open_result: int = 29, signal_result: int = 0) -> None:
                self.pidfd_open = FakeFunction(open_result)
                self.pidfd_send_signal = FakeFunction(signal_result)

        libc = FakeLibc()
        with (
            patch.object(stop.os, "pidfd_open", None, create=True),
            patch.object(stop.sys, "platform", "linux"),
            patch.object(ctypes, "CDLL", return_value=libc),
        ):
            self.assertEqual(stop._open_process_handle(41), 29)

        with (
            patch.object(stop.os, "pidfd_open", None, create=True),
            patch.object(stop.sys, "platform", "darwin"),
        ):
            self.assertIsNone(stop._open_process_handle(41))

        with (
            patch.object(stop.signal, "pidfd_send_signal", None, create=True),
            patch.object(ctypes, "CDLL", return_value=libc),
        ):
            stop._signal_process_handle(29, stop.signal.SIGTERM)

        failing_libc = FakeLibc(signal_result=-1)
        with (
            patch.object(stop.signal, "pidfd_send_signal", None, create=True),
            patch.object(ctypes, "CDLL", return_value=failing_libc),
            patch.object(ctypes, "get_errno", return_value=1),
            self.assertRaises(OSError),
        ):
            stop._signal_process_handle(29, stop.signal.SIGTERM)

    def test_scheduler_state_accepts_cross_platform_process_identity(self) -> None:
        from praxist.cli import stop
        from praxist.plugins.workflow_stages.research_loop.backend import protected_pids

        entry = protected_pids.ProtectedEntry(
            pid=6101,
            pgid=6101,
            pid_start_time="ps:Mon Aug 11 12:00:00 2026",
        )
        with (
            patch.object(protected_pids, "list_all_protected", return_value=[entry]),
            patch.object(protected_pids, "_entry_process_identity_matches", return_value=True),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.getpgid", return_value=6101),
        ):
            observed = stop._verified_scheduler_entries(Path("/run"))

        self.assertEqual(observed, [entry])

    def test_scheduler_state_uses_identity_bound_live_authority(self) -> None:
        from praxist.cli import stop

        ps_rows = {6101: (1, "00:09", "python evaluator.py")}
        with (
            patch("praxist.cli.stop._verified_scheduler_entries", return_value=[]),
            patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler_client.scheduler_active_process_groups",
                return_value={6100: (6101, "ps:stable")},
            ),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.getpgid", return_value=6100),
            patch("praxist.cli.stop.process_start_token", return_value="ps:stable"),
        ):
            targets, groups = stop._scheduler_owned_state(Path("/run"), ps_rows)

        self.assertEqual(targets, {6101})
        self.assertEqual(groups, {6100: {6101: "ps:stable"}})

    def test_scheduler_state_rejects_reused_rpc_launcher_identity(self) -> None:
        from praxist.cli import stop

        ps_rows = {6101: (1, "00:09", "python evaluator.py")}
        with (
            patch("praxist.cli.stop._verified_scheduler_entries", return_value=[]),
            patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler_client.scheduler_active_process_groups",
                return_value={6100: (6101, "ps:former")},
            ),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.getpgid", return_value=6100),
            patch("praxist.cli.stop.process_start_token", return_value="ps:replacement"),
        ):
            targets, groups = stop._scheduler_owned_state(Path("/run"), ps_rows)

        self.assertEqual(targets, set())
        self.assertEqual(groups, {})

    def test_run_owned_discovery_does_not_bind_identity_after_ownership_check(self) -> None:
        from praxist.cli import stop

        run_dir = Path("/run")
        rows = {6200: (1, "00:01", "python evaluator.py")}
        with (
            patch(
                "praxist.cli.stop._read_identity_bound_ps_snapshot",
                return_value=(rows, {}),
            ),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
            patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
            patch(
                "praxist.cli.stop._run_dir_from_process_environment",
                return_value=run_dir,
            ),
            patch("praxist.cli.stop.process_start_token", return_value="ps:replacement"),
        ):
            tokens, groups = stop._discover_run_owned_processes(run_dir)

        self.assertEqual(tokens, {6200: ""})
        self.assertEqual(groups, {})
        with patch("praxist.cli.stop.pid_is_alive", return_value=True):
            self.assertEqual(stop._live_process_instances(tokens, tokens), {6200})

    def test_stop_reports_unverified_legacy_group_instead_of_false_success(self) -> None:
        from praxist.cli import registry, stop
        from praxist.plugins.workflow_stages.research_loop.backend import protected_pids

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            run_dir.mkdir()
            run_id = _write_entry(run_dir=str(run_dir))
            orphan = protected_pids.ProtectedEntry(
                pid=6100,
                pgid=6100,
                pid_start_time="ps:former-launcher",
            )
            with (
                patch(
                    "praxist.cli.stop._read_identity_bound_ps_snapshot",
                    return_value=({}, {}),
                ),
                patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
                patch("praxist.cli.stop._drain_late_run_processes", return_value={}),
                patch.object(protected_pids, "list_all_protected", return_value=[orphan]),
                patch.object(protected_pids, "_is_process_group_alive", return_value=True),
                patch("praxist.cli.stop.pid_is_alive", return_value=False),
            ):
                outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertEqual(outcome.failed_run_ids, [run_id])
        self.assertTrue(any("could not be identity-verified" in w for w in outcome.warnings))
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_live_legacy_launcher_without_start_identity_remains_unresolved(self) -> None:
        from praxist.cli import stop
        from praxist.plugins.workflow_stages.research_loop.backend import protected_pids

        entry = protected_pids.ProtectedEntry(pid=6100, pgid=6100, pid_start_time=None)
        outcome = stop.StopOutcome(run_id="run_missing_identity")
        with (
            patch.object(protected_pids, "list_all_protected", return_value=[entry]),
            patch.object(protected_pids, "_is_process_group_alive", return_value=True),
            patch.object(protected_pids, "_entry_process_identity_matches", return_value=True),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
        ):
            stop._record_unresolved_protected_groups(
                "run_missing_identity",
                Path("/run"),
                outcome,
            )

        self.assertEqual(outcome.failed_run_ids, ["run_missing_identity"])

    def test_process_run_identity_uses_exact_inherited_environment(self) -> None:
        from praxist.cli import stop

        payload = b"PATH=/bin\0PRAXIST_RUN_DIR=/tmp/exact-run\0"
        with patch.object(Path, "read_bytes", return_value=payload):
            self.assertEqual(
                stop._run_dir_from_process_environment(123),
                Path("/tmp/exact-run").resolve(),
            )

    def test_cwd_fallback_is_limited_to_run_owned_peer_workspaces(self) -> None:
        from praxist.cli import stop

        run_dir = Path("/tmp/exact-run").resolve()
        with patch(
            "praxist.cli.stop.os.readlink",
            return_value=str(run_dir / "peer_workspaces" / "gen2_peer3"),
        ):
            self.assertTrue(stop._process_cwd_is_run_workspace(123, run_dir))
        with patch("praxist.cli.stop.os.readlink", return_value=str(run_dir)):
            self.assertFalse(stop._process_cwd_is_run_workspace(123, run_dir))

    def test_stop_does_not_target_independent_monitor_processes(self) -> None:
        from praxist.cli import stop

        run_id = _write_entry()
        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
            7000: (1, "00:01", f"python -m praxist.cli.monitor --run-id {run_id}"),
            8000: (1, "00:01", f"praxist --monitor --run-id {run_id}"),
        }
        alive = {5000, 7000, 8000}
        signalled: list[int] = []

        def fake_kill(pid: int, _sig: int) -> None:
            signalled.append(pid)
            alive.discard(pid)

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", side_effect=lambda pid: pid in alive),
            patch("praxist.cli.stop.os.kill", side_effect=fake_kill),
            patch("praxist.cli.stop.time.sleep"),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertEqual(outcome.matched_pids, [5000])
        self.assertEqual(outcome.monitor_sessions, [])
        self.assertEqual(outcome.monitor_stopped_sessions, [])
        self.assertNotIn(7000, signalled)
        self.assertNotIn(8000, signalled)

    def test_dead_root_still_stops_scheduler_owned_process_group(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(pid=5000)
        ps_rows = {6001: (1, "00:02", "python evaluator.py")}
        alive = {6001}
        group_alive = {6001}
        group_signals: list[tuple[int, int]] = []
        pid_signals: list[tuple[int, int]] = []

        def fake_alive(pid: int) -> bool:
            return pid in alive

        def fake_kill(pid: int, sig: int) -> None:
            pid_signals.append((pid, sig))
            alive.discard(pid)

        def fake_killpg(pgid: int, sig: int) -> None:
            if sig == 0:
                if pgid not in group_alive:
                    raise ProcessLookupError
                return
            group_signals.append((pgid, sig))
            group_alive.discard(pgid)

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", side_effect=fake_alive),
            patch("praxist.cli.stop.os.kill", side_effect=fake_kill),
            patch("praxist.cli.stop.os.killpg", side_effect=fake_killpg),
            patch("praxist.cli.stop.os.getpgid", return_value=6001),
            patch("praxist.cli.stop.process_start_token", return_value="proc:123"),
            patch("praxist.cli.stop.time.sleep"),
            patch(
                "praxist.cli.stop._scheduler_owned_state",
                return_value=({6001}, {6001: {6001: "proc:123"}}),
            ),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertEqual(outcome.matched_pids, [])
        self.assertEqual(outcome.descendant_pids, [6001])
        self.assertIn((6001, 15), pid_signals)
        self.assertNotIn((6001, 15), group_signals)
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)

    def test_scheduler_groups_require_live_matching_leader_identity(self) -> None:
        from praxist.cli import stop
        from praxist.plugins.workflow_stages.research_loop.backend.protected_pids import (
            ProtectedEntry,
        )

        entry = ProtectedEntry(
            pid=6001,
            pgid=6001,
            pid_start_time=123,
            peer_id="gen0_peer1",
        )
        manifest_module = "praxist.plugins.workflow_stages.research_loop.backend.protected_pids"
        with (
            patch(f"{manifest_module}.list_all_protected", return_value=[entry]),
            patch(f"{manifest_module}._entry_process_identity_matches", return_value=True),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.getpgid", return_value=6001),
            patch(
                "praxist.cli.stop.read_ps_table",
                return_value={6001: (1, "", "")},
            ),
        ):
            self.assertEqual(
                stop._scheduler_owned_targets(Path("/tmp/run"), {6001: (1, "", "")}),
                {6001},
            )
            self.assertEqual(
                stop._scheduler_owned_process_groups(Path("/tmp/run")),
                {6001},
            )

        with (
            patch(f"{manifest_module}.list_all_protected", return_value=[entry]),
            patch(f"{manifest_module}._entry_process_identity_matches", return_value=False),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.os.getpgid") as getpgid,
        ):
            self.assertEqual(
                stop._scheduler_owned_targets(
                    Path("/tmp/run"),
                    {6001: (1, "", "")},
                ),
                set(),
            )
            self.assertEqual(
                stop._scheduler_owned_process_groups(Path("/tmp/run")),
                set(),
            )
            getpgid.assert_not_called()

    def test_scheduler_state_rejects_group_after_recorded_leader_exits(self) -> None:
        from praxist.cli import stop
        from praxist.plugins.workflow_stages.research_loop.backend.protected_pids import (
            ProtectedEntry,
        )

        entry = ProtectedEntry(
            pid=6001,
            pgid=6001,
            pid_start_time=123,
            peer_id="gen0_peer1",
        )
        manifest_module = "praxist.plugins.workflow_stages.research_loop.backend.protected_pids"
        with (
            patch(f"{manifest_module}.list_all_protected", return_value=[entry]),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.getpgid", return_value=6001),
            patch("praxist.cli.stop.process_start_token", return_value="proc:child"),
        ):
            targets, groups = stop._scheduler_owned_state(
                Path("/tmp/run"),
                {6002: (1, "", "python evaluator.py")},
            )

        self.assertEqual(targets, set())
        self.assertEqual(groups, {})

    def test_scheduler_state_rejects_legacy_member_without_start_token(self) -> None:
        from praxist.cli import stop
        from praxist.plugins.workflow_stages.research_loop.backend.protected_pids import (
            ProtectedEntry,
        )

        entry = ProtectedEntry(
            pid=6001,
            pgid=6001,
            pid_start_time=None,
            peer_id="gen0_peer1",
        )
        manifest_module = "praxist.plugins.workflow_stages.research_loop.backend.protected_pids"
        with (
            patch(f"{manifest_module}.list_all_protected", return_value=[entry]),
            patch(f"{manifest_module}._entry_process_identity_matches", return_value=True),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.getpgid", return_value=6001),
            patch("praxist.cli.stop.process_start_token", return_value=""),
        ):
            targets, groups = stop._scheduler_owned_state(
                Path("/tmp/run"),
                {6001: (1, "", "python evaluator.py")},
            )

        self.assertEqual(targets, set())
        self.assertEqual(groups, {})

    def test_corrupt_scheduler_manifest_is_treated_as_unverified(self) -> None:
        from praxist.cli import stop

        manifest_module = "praxist.plugins.workflow_stages.research_loop.backend.protected_pids"
        with patch(
            f"{manifest_module}.list_all_protected",
            side_effect=OSError("corrupt manifest"),
        ):
            entries = stop._verified_scheduler_entries(Path("/tmp/run"))

        self.assertEqual(entries, [])

    def test_toctou_mismatch_skips_signal_without_claiming_success(self) -> None:
        """Registry's pid is alive but ``ps`` shows a different command."""
        from praxist.cli import registry, stop

        run_id = _write_entry()
        # ps row claims pid 5000 is `/bin/bash` now (PID was recycled).
        ps_rows = {5000: (1, "00:01", "/bin/bash")}
        killed: list[tuple[int, int]] = []

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill", side_effect=lambda p, s: killed.append((p, s))),
            patch("praxist.cli.stop.time.sleep"),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertEqual(killed, [])  # no signals
        self.assertEqual(outcome.matched_pids, [])
        self.assertTrue(any("TOCTOU" in w for w in outcome.warnings))
        self.assertEqual(outcome.remaining_pids, [5000])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_reused_pid_after_sigterm_is_not_sent_sigkill(self) -> None:
        """Escalation must not signal a new process that reused the target PID."""
        import signal as _sig

        from praxist.cli import registry, stop

        run_id = _write_entry(extra={"process_start_token": "proc:old"})
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        sent: list[tuple[int, int]] = []
        tokens = iter(("proc:old", "proc:old", "proc:old", "proc:new", "proc:new"))

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.process_identity_matches", return_value=True),
            patch("praxist.cli.stop.process_start_token", side_effect=lambda _pid: next(tokens)),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch(
                "praxist.cli.stop.os.kill",
                side_effect=lambda pid, sig: sent.append((pid, sig)),
            ),
            patch("praxist.cli.stop._discover_run_owned_processes", return_value=({}, {})),
            patch("praxist.cli.stop.time.sleep"),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertEqual(sent, [(5000, _sig.SIGTERM)])
        self.assertEqual(outcome.killed_pids, [])
        self.assertEqual(outcome.remaining_pids, [])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)

    def test_ps_target_reused_during_discovery_is_not_signalled(self) -> None:
        from praxist.cli import stop

        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
            5001: (5000, "00:09", "python evaluator.py"),
        }
        sent: list[tuple[int, int]] = []
        root_tokens = iter(("proc:old", "proc:new"))

        def token_for(pid: int) -> str:
            return next(root_tokens) if pid == 5000 else "proc:child"

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch(
                "praxist.cli.stop.process_start_token",
                side_effect=token_for,
            ),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
            patch(
                "praxist.cli.stop.os.kill",
                side_effect=lambda pid, sig: sent.append((pid, sig)),
            ),
            patch("praxist.cli.stop.time.sleep"),
        ):
            outcome = stop.stop_all(
                scope=stop.SCOPE_PS,
                grace_seconds=0.0,
            )

        self.assertEqual(sent, [])
        self.assertEqual(outcome.remaining_pids, [5000])
        self.assertEqual(outcome.descendant_pids, [])

    def test_recorded_root_token_does_not_use_command_fallback_when_unknown(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(extra={"process_start_token": "proc:expected"})
        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
            5001: (5000, "00:09", "python evaluator.py"),
        }
        sent: list[tuple[int, int]] = []
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch(
                "praxist.cli.stop.process_start_token",
                side_effect=lambda pid: "" if pid == 5000 else "proc:child",
            ),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch(
                "praxist.cli.stop.os.kill",
                side_effect=lambda pid, sig: sent.append((pid, sig)),
            ),
            patch("praxist.cli.stop.time.sleep"),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertEqual(sent, [])
        self.assertEqual(outcome.remaining_pids, [5000])
        self.assertEqual(outcome.descendant_pids, [])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_reused_process_group_leader_is_not_sent_later_signal(self) -> None:
        import signal as _sig

        from praxist.cli import stop

        outcome = stop.StopOutcome(run_id="run_group_reuse")
        signals: list[tuple[int, int]] = []
        members = {6001: {6001: "proc:old"}}
        with (
            patch(
                "praxist.cli.stop.read_ps_table",
                return_value={6001: (1, "00:01", "python evaluator.py")},
            ),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch(
                "praxist.cli.stop.process_start_token",
                side_effect=("proc:old", "proc:new", "proc:new"),
            ),
            patch("praxist.cli.stop.os.getpgid", return_value=6001),
            patch(
                "praxist.cli.stop.os.killpg",
                side_effect=lambda pgid, sig: signals.append((pgid, sig)),
            ),
        ):
            stop._signal_process_groups(
                {6001},
                _sig.SIGTERM,
                outcome,
                group_members=members,
            )
            stop._signal_process_groups(
                {6001},
                _sig.SIGKILL,
                outcome,
                group_members=members,
            )

        self.assertEqual(
            signals,
            [(6001, 0)],
        )

    def test_signal_set_uses_identity_bound_process_handle_when_available(self) -> None:
        import signal as _sig

        from praxist.cli import stop

        outcome = stop.StopOutcome(run_id="run_pidfd")
        with (
            patch("praxist.cli.stop._open_process_handle", return_value=77),
            patch("praxist.cli.stop._signal_process_handle") as send_handle,
            patch("praxist.cli.stop.process_start_token", return_value="proc:stable"),
            patch("praxist.cli.stop.os.kill") as numeric_kill,
            patch("praxist.cli.stop.os.close"),
        ):
            stop._signal_set(
                {6001},
                _sig.SIGTERM,
                outcome,
                process_tokens={6001: "proc:stable"},
            )

        send_handle.assert_called_once_with(77, _sig.SIGTERM)
        self.assertNotIn((6001, _sig.SIGTERM), [call.args for call in numeric_kill.mock_calls])

    def test_mixed_identity_process_group_skips_group_signal(self) -> None:
        from praxist.cli import stop

        outcome = stop.StopOutcome(run_id="run_mixed_group")
        group_signals: list[tuple[int, int]] = []
        members = {
            7000: {
                7001: "proc:verified",
                7002: "proc:expected",
            }
        }

        def token_for(pid: int) -> str:
            return "proc:verified" if pid == 7001 else ""

        with (
            patch(
                "praxist.cli.stop.read_ps_table",
                return_value={
                    7001: (1, "00:01", "python evaluator.py"),
                    7002: (1, "00:01", "python evaluator.py"),
                },
            ),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.process_start_token", side_effect=token_for),
            patch("praxist.cli.stop.os.getpgid", return_value=7000),
            patch(
                "praxist.cli.stop.os.killpg",
                side_effect=lambda pgid, sig: group_signals.append((pgid, sig)),
            ),
        ):
            stop._signal_process_groups(
                {7000},
                15,
                outcome,
                group_members=members,
            )

        self.assertEqual(group_signals, [])
        self.assertEqual(outcome.remaining_pids, [7002])
        self.assertTrue(any("skipping the group signal" in item for item in outcome.warnings))

    def test_unknown_member_identity_is_left_running_and_reported(self) -> None:
        from praxist.cli import stop

        outcome = stop.StopOutcome(run_id="run_unknown_member")
        sent: list[tuple[int, int]] = []
        with (
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.process_start_token", return_value=""),
            patch(
                "praxist.cli.stop.os.kill",
                side_effect=lambda pid, sig: sent.append((pid, sig)),
            ),
        ):
            stop._signal_set(
                {6002},
                15,
                outcome,
                process_tokens={6002: "proc:expected"},
            )

        self.assertEqual(sent, [])
        self.assertEqual(outcome.remaining_pids, [6002])
        self.assertTrue(any("cannot verify process identity" in item for item in outcome.warnings))

    def test_unknown_root_identity_uses_registry_command_fallback(self) -> None:
        from praxist.cli import registry, stop

        entry = registry.RegistryEntry(**_entry_kwargs(pid=5000))
        outcome = stop.StopOutcome(run_id=entry.run_id)
        sent: list[tuple[int, int]] = []
        ps_rows = {5000: (1, "00:01", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.process_start_token", return_value=""),
            patch("praxist.cli.stop.process_identity_matches", return_value=None),
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch(
                "praxist.cli.stop.os.kill",
                side_effect=lambda pid, sig: sent.append((pid, sig)),
            ),
        ):
            stop._signal_set(
                {5000},
                15,
                outcome,
                process_tokens={5000: "proc:expected"},
                fallback_entries={5000: entry},
            )

        self.assertEqual(sent, [(5000, 15)])
        self.assertEqual(outcome.remaining_pids, [])

    def test_strong_root_identity_does_not_depend_on_command_text(self) -> None:
        from praxist.cli import registry, stop

        entry = registry.RegistryEntry(**_entry_kwargs(extra={"process_start_token": "proc:live"}))
        outcome = stop.StopOutcome(run_id=entry.run_id)
        with (
            patch("praxist.cli.stop.process_identity_matches", return_value=True),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
        ):
            root = stop._validate_registry_root(
                entry,
                {entry.pid: (1, "00:01", "renamed-controller")},
                outcome,
            )

        self.assertEqual(root, entry.pid)
        self.assertEqual(outcome.remaining_pids, [])

    def test_dead_pid_marks_stopped_without_signalling(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill") as kill_mock,
            patch("praxist.cli.stop.time.sleep"),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)
        kill_mock.assert_not_called()
        self.assertEqual(outcome.matched_pids, [])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)

    def test_epoch_mismatch_preserves_registry_and_reports_failure(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry()
        with (
            patch("praxist.cli.stop.entry_process_epoch_matches", return_value=False),
            patch("praxist.cli.stop.os.kill") as kill_mock,
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        kill_mock.assert_not_called()
        self.assertEqual(outcome.matched_pids, [])
        self.assertEqual(outcome.failed_run_ids, [run_id])
        self.assertTrue(any("state was preserved" in warning for warning in outcome.warnings))
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_non_local_entry_is_rejected_before_process_discovery(self) -> None:
        from praxist.cli import stop

        run_id = _write_entry(extra={"hostname": "remote.example"})
        with (
            patch("praxist.cli.stop.entry_is_local", return_value=False),
            patch("praxist.cli.stop._read_identity_bound_ps_snapshot") as snapshot,
            patch("praxist.cli.stop.os.kill") as kill,
            self.assertRaisesRegex(stop.StopError, "remote.example"),
        ):
            stop.stop_run(run_id=run_id, grace_seconds=0.0)

        snapshot.assert_not_called()
        kill.assert_not_called()

    def test_scheduler_fence_failures_warn_but_do_not_block_stop(self) -> None:
        from praxist.cli import registry, stop

        def prepare_run_dir(name: str) -> Path:
            run_dir = Path(self.tmp.name) / name
            endpoint = run_dir / "resource_scheduler" / "endpoint.json"
            endpoint.parent.mkdir(parents=True)
            endpoint.write_text("{}\n", encoding="utf-8")
            return run_dir

        unavailable_dir = prepare_run_dir("fence_unavailable")
        failing_dir = prepare_run_dir("fence_failing")
        unavailable_id = _write_entry(
            run_id="run_fence_unavailable",
            pid=5100,
            run_dir=str(unavailable_dir),
        )
        failing_id = _write_entry(
            run_id="run_fence_failing",
            pid=5200,
            run_dir=str(failing_dir),
        )

        def freeze(run_dir: Path, reason: str) -> bool:
            self.assertEqual(reason, "praxist_stop")
            if run_dir == unavailable_dir:
                return False
            raise RuntimeError("scheduler unavailable")

        scheduler_client = (
            "praxist.plugins.workflow_stages.research_loop.backend."
            "experiment_scheduler_client.freeze_all_for_run"
        )
        with (
            patch("praxist.cli.stop.entry_process_epoch_matches", return_value=True),
            patch(
                "praxist.cli.stop._read_identity_bound_ps_snapshot",
                return_value=({}, {}),
            ),
            patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch(scheduler_client, side_effect=freeze) as freeze_mock,
            patch("praxist.cli.stop.os.kill") as kill,
        ):
            unavailable = stop.stop_run(run_id=unavailable_id, grace_seconds=0.0)
            failing = stop.stop_run(run_id=failing_id, grace_seconds=0.0)

        self.assertEqual(freeze_mock.call_count, 2)
        self.assertIn("central scheduler stop fence was unavailable", unavailable.warnings)
        self.assertTrue(any("scheduler unavailable" in warning for warning in failing.warnings))
        kill.assert_not_called()
        self.assertEqual(registry.read_entry(unavailable_id).state, registry.STATE_STOPPED)
        self.assertEqual(registry.read_entry(failing_id).state, registry.STATE_STOPPED)

    def test_invalid_grace_is_rejected_before_registry_access(self) -> None:
        from praxist.cli import stop

        with (
            patch("praxist.cli.stop.entry_lock") as lock,
            self.assertRaisesRegex(stop.StopError, "finite non-negative"),
        ):
            stop.stop_run(run_id="never-read", grace_seconds=-0.01)

        lock.assert_not_called()

    def test_registry_lock_error_is_translated_to_stop_error(self) -> None:
        from praxist.cli import registry, stop

        with (
            patch(
                "praxist.cli.stop.entry_lock",
                side_effect=registry.RegistryError("lock unavailable"),
            ),
            self.assertRaisesRegex(stop.StopError, "lock unavailable") as raised,
        ):
            stop.stop_run(run_id="run_locked", grace_seconds=0.0)

        self.assertIsInstance(raised.exception.__cause__, registry.RegistryError)

    def test_grace_expiry_escalates_to_sigkill(self) -> None:
        import signal as _sig

        from praxist.cli import stop

        run_id = _write_entry()
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        killed: list[tuple[int, int]] = []

        # Pid stays alive through TERM grace, dies after KILL.
        kill_count = {"value": 0}

        def fake_alive(pid: int) -> bool:
            return kill_count["value"] < 2

        def fake_kill(pid: int, sig: int) -> None:
            killed.append((pid, sig))
            if sig == _sig.SIGKILL:
                kill_count["value"] = 2

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", side_effect=fake_alive),
            patch("praxist.cli.stop.os.kill", side_effect=fake_kill),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.time.monotonic", side_effect=[0.0, 0.1, 1.0]),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.05)

        # TERM then KILL.
        sigs = [s for _p, s in killed]
        self.assertIn(_sig.SIGTERM, sigs)
        self.assertIn(_sig.SIGKILL, sigs)
        self.assertEqual(outcome.killed_pids, [5000])
        self.assertEqual(outcome.remaining_pids, [])

    def test_dry_run_does_not_signal(self) -> None:
        from praxist.cli import stop

        run_id = _write_entry()
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill") as kill_mock,
            patch("praxist.cli.stop.time.sleep"),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0, dry_run=True)
        kill_mock.assert_not_called()
        self.assertEqual(outcome.matched_pids, [5000])
        self.assertTrue(outcome.dry_run)

    def test_missing_registry_entry_raises_stop_error(self) -> None:
        from praxist.cli import stop

        with self.assertRaises(stop.StopError):
            stop.stop_run(run_id="does_not_exist", grace_seconds=0.0)

    def test_stop_rereads_entry_after_acquiring_lifecycle_lock(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(pid=5000)
        current = registry.read_entry(run_id)
        replacement = registry.RegistryEntry(
            **_entry_kwargs(
                pid=6000,
                run_dir=current.run_dir,
                log_file=current.log_file,
            )
        )

        @contextmanager
        def replace_before_lock_body(_run_id: str):
            registry.write_entry(replacement)
            yield

        ps_rows = {6000: (1, "00:01", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.entry_lock", side_effect=replace_before_lock_body),
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)

        self.assertEqual(outcome.matched_pids, [6000])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)


class StopAllTest(_StateDirMixin, unittest.TestCase):
    """``praxist stop --all`` and scope flags."""

    def _ps_table_with_praxist_processes(self) -> dict[int, tuple[int, str, str]]:
        # 5001 is intentionally a *non-pattern-matching* child of 5000 so
        # the test exercises descendant walking rather than the Praxist-pattern
        # ps-scan path.
        return {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
            5001: (5000, "00:09", "python /tmp/helper_script.py"),
            6000: (1, "00:01", "python -m praxist.run run --task-path /other"),
            7000: (1, "00:30", "/sbin/launchd"),  # unrelated, must not match
        }

    def test_union_scope_matches_registry_and_ps_scan(self) -> None:
        from praxist.cli import registry, stop

        # registry: run with pid 5000
        run_id = _write_entry()
        # ps shows 5000 (registry hit), 6000 (ps-only praxist.run), 7000 (skip)
        ps_rows = self._ps_table_with_praxist_processes()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_UNION, grace_seconds=0.0)

        # Both 5000 (registry root) and 6000 (ps-only) targeted; 7000 skipped.
        self.assertIn(5000, outcome.matched_pids)
        self.assertIn(6000, outcome.matched_pids)
        self.assertNotIn(7000, outcome.matched_pids)
        # Descendant 5001 reached via the ps table.
        self.assertIn(5001, outcome.descendant_pids)
        # Registry entry for 5000 stamped stopped.
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)

    def test_union_does_not_readd_registry_pid_with_rejected_strong_identity(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(extra={"process_start_token": "proc:old"})
        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
            5001: (5000, "00:09", "python evaluator.py"),
        }
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.process_identity_matches", return_value=False),
            patch(
                "praxist.cli.stop.process_start_token",
                side_effect=lambda pid: f"proc:new:{pid}",
            ),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill") as kill,
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_UNION, grace_seconds=0.0)

        kill.assert_not_called()
        self.assertEqual(outcome.matched_pids, [])
        self.assertEqual(outcome.descendant_pids, [])
        self.assertEqual(outcome.remaining_pids, [5000])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_registry_scope_leaves_monitor_running_when_run_root_is_dead(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(pid=99999)
        ps_rows = {6000: (1, "00:09", f"python -m praxist.cli.monitor --run-id {run_id}")}

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch(
                "praxist.cli.stop.pid_is_alive",
                side_effect=lambda pid: pid == 6000,
            ),
            patch("praxist.cli.stop.os.kill") as kill,
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)

        self.assertEqual(outcome.matched_pids, [])
        self.assertEqual(outcome.monitor_sessions, [])
        self.assertEqual(outcome.monitor_stopped_sessions, [])
        kill.assert_not_called()
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)

    def test_registry_only_scope_ignores_ps_only_matches(self) -> None:
        from praxist.cli import stop

        _write_entry()
        ps_rows = self._ps_table_with_praxist_processes()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)

        self.assertIn(5000, outcome.matched_pids)
        self.assertNotIn(6000, outcome.matched_pids)

    def test_ps_scan_only_scope_ignores_registry_entries(self) -> None:
        from praxist.cli import stop

        # The registry entry has pid 5000; but in ps-scan-only mode it
        # should still be matched (pid 5000 also appears in ps as
        # praxist.run). Verify a registry-only pid (one absent
        # from ps) is NOT included.
        _write_entry(run_id="run_registry_only", pid=99999)
        ps_rows = self._ps_table_with_praxist_processes()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_PS, grace_seconds=0.0)

        # 5000 + 6000 from ps-scan; 99999 (registry-only) not present.
        self.assertIn(5000, outcome.matched_pids)
        self.assertIn(6000, outcome.matched_pids)
        self.assertNotIn(99999, outcome.matched_pids)

    def test_ps_scan_does_not_signal_unowned_task_shaped_processes(self) -> None:
        from praxist.cli import stop

        ps_rows = {
            6100: (1, "00:10", "python /other/project/train.py --epochs 3"),
            6101: (1, "00:09", "python /other/project/run_benchmark.py"),
            6102: (1, "00:08", "claude --output-format stream-json"),
        }
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.os.kill") as kill_mock,
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_PS, grace_seconds=0.0)

        self.assertEqual(outcome.matched_pids, [])
        self.assertEqual(outcome.descendant_pids, [])
        kill_mock.assert_not_called()

    def test_dry_run_does_not_signal_or_update_registry(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry()
        ps_rows = self._ps_table_with_praxist_processes()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill") as kill_mock,
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_UNION, grace_seconds=0.0, dry_run=True)
        kill_mock.assert_not_called()
        self.assertTrue(outcome.dry_run)
        # Registry entry remains running.
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_stop_all_rereads_entries_after_acquiring_lifecycle_locks(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(pid=5000)
        current = registry.read_entry(run_id)
        replacement = registry.RegistryEntry(
            **_entry_kwargs(
                pid=6000,
                run_dir=current.run_dir,
                log_file=current.log_file,
            )
        )

        @contextmanager
        def replace_before_lock_body(_run_id: str):
            registry.write_entry(replacement)
            yield

        ps_rows = {6000: (1, "00:01", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.entry_lock", side_effect=replace_before_lock_body),
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)

        self.assertEqual(outcome.matched_pids, [6000])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)

    def test_stop_all_enumerates_after_acquiring_registry_lock(self) -> None:
        from praxist.cli import registry, stop

        run_id = "run_created_before_global_lock_body"
        run_dir = Path(self.tmp.name) / "run_dirs" / run_id
        run_dir.mkdir(parents=True)
        entry = registry.RegistryEntry(
            **_entry_kwargs(
                run_id=run_id,
                pid=6000,
                run_dir=str(run_dir),
                log_file=str(run_dir / "log"),
            )
        )

        @contextmanager
        def create_before_lock_body():
            registry.write_entry(entry)
            yield

        ps_rows = {6000: (1, "00:01", "python -m praxist.run run --task-path /t")}
        with (
            patch(
                "praxist.cli.stop.registry_lock",
                side_effect=create_before_lock_body,
            ),
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)

        self.assertEqual(outcome.matched_pids, [6000])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_STOPPED)

    def test_remote_and_prior_boot_entries_are_handled_conservatively(self) -> None:
        from praxist.cli import registry, stop

        remote_id = _write_entry(
            run_id="run_remote",
            pid=5100,
            extra={"hostname": "remote.example"},
        )
        prior_ok_id = _write_entry(run_id="run_prior_ok", pid=5200)
        prior_fail_id = _write_entry(run_id="run_prior_fail", pid=5300)

        with (
            patch(
                "praxist.cli.stop.entry_is_local",
                side_effect=lambda entry: entry.run_id != remote_id,
            ),
            patch("praxist.cli.stop.entry_process_epoch_matches", return_value=False),
            patch(
                "praxist.cli.stop._read_identity_bound_ps_snapshot",
                return_value=({}, {}),
            ),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
            patch("praxist.cli.stop._scheduler_owned_state") as scheduler_state,
            patch("praxist.cli.stop.os.kill") as kill,
        ):
            outcome = stop.stop_all(
                scope=stop.SCOPE_REGISTRY,
                grace_seconds=0.0,
                now_iso="2026-07-23T12:00:00+00:00",
            )

        scheduler_state.assert_not_called()
        kill.assert_not_called()
        self.assertTrue(any("remote.example" in warning for warning in outcome.warnings))
        self.assertTrue(any("state was preserved" in warning for warning in outcome.warnings))
        self.assertCountEqual(outcome.failed_run_ids, [prior_ok_id, prior_fail_id])
        self.assertEqual(registry.read_entry(remote_id).state, registry.STATE_RUNNING)
        self.assertEqual(registry.read_entry(prior_ok_id).state, registry.STATE_RUNNING)
        self.assertEqual(registry.read_entry(prior_fail_id).state, registry.STATE_RUNNING)

    def test_stop_all_reports_each_scheduler_fence_failure(self) -> None:
        from praxist.cli import registry, stop

        def prepare_run_dir(name: str) -> Path:
            run_dir = Path(self.tmp.name) / name
            endpoint = run_dir / "resource_scheduler" / "endpoint.json"
            endpoint.parent.mkdir(parents=True)
            endpoint.write_text("{}\n", encoding="utf-8")
            return run_dir

        unavailable_dir = prepare_run_dir("all_fence_unavailable")
        failing_dir = prepare_run_dir("all_fence_failing")
        unavailable_id = _write_entry(
            run_id="run_all_fence_unavailable",
            pid=6100,
            run_dir=str(unavailable_dir),
        )
        failing_id = _write_entry(
            run_id="run_all_fence_failing",
            pid=6200,
            run_dir=str(failing_dir),
        )

        def freeze(run_dir: Path, reason: str) -> bool:
            self.assertEqual(reason, "praxist_stop_all")
            if run_dir == unavailable_dir:
                return False
            raise RuntimeError("scheduler RPC failed")

        scheduler_client = (
            "praxist.plugins.workflow_stages.research_loop.backend."
            "experiment_scheduler_client.freeze_all_for_run"
        )
        with (
            patch("praxist.cli.stop.entry_is_local", return_value=True),
            patch("praxist.cli.stop.entry_process_epoch_matches", return_value=True),
            patch(
                "praxist.cli.stop._read_identity_bound_ps_snapshot",
                return_value=({}, {}),
            ),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
            patch(scheduler_client, side_effect=freeze) as freeze_mock,
            patch("praxist.cli.stop.os.kill") as kill,
        ):
            outcome = stop.stop_all(
                scope=stop.SCOPE_REGISTRY,
                grace_seconds=0.0,
            )

        self.assertEqual(freeze_mock.call_count, 2)
        self.assertTrue(
            any(
                unavailable_id in warning and "unavailable" in warning
                for warning in outcome.warnings
            )
        )
        self.assertTrue(
            any(
                failing_id in warning and "scheduler RPC failed" in warning
                for warning in outcome.warnings
            )
        )
        kill.assert_not_called()
        self.assertEqual(registry.read_entry(unavailable_id).state, registry.STATE_STOPPED)
        self.assertEqual(registry.read_entry(failing_id).state, registry.STATE_STOPPED)

    def test_stop_all_skips_run_whose_admission_cannot_close(self) -> None:
        from praxist.cli import registry, stop

        with tempfile.TemporaryDirectory() as td:
            blocked_dir = Path(td) / "blocked"
            fenced_dir = Path(td) / "fenced"
            blocked_dir.mkdir()
            fenced_dir.mkdir()
            blocked_id = _write_entry(
                run_id="run_unfenced",
                pid=6100,
                run_dir=str(blocked_dir),
            )
            fenced_id = _write_entry(
                run_id="run_fenced",
                pid=6200,
                run_dir=str(fenced_dir),
            )
            ps_rows = {
                6100: (1, "00:10", "python -m praxist.run run --task-path /t"),
                6101: (6100, "00:09", "python blocked_eval.py"),
                6200: (1, "00:10", "python -m praxist.run run --task-path /t"),
            }
            alive = {6100, 6101, 6200}
            signals: list[tuple[int, int]] = []

            def signal_process(pid: int, sig: int) -> None:
                signals.append((pid, sig))
                alive.discard(pid)

            def write_fence(run_dir: Path, _outcome, *, source: str) -> bool:
                self.assertEqual(source, "praxist_stop_all")
                return Path(run_dir) == fenced_dir

            with (
                patch(
                    "praxist.cli.stop._write_run_shutdown_fence",
                    side_effect=write_fence,
                ),
                patch(
                    "praxist.cli.stop._read_identity_bound_ps_snapshot",
                    return_value=(ps_rows, {pid: f"proc:{pid}" for pid in ps_rows}),
                ),
                patch("praxist.cli.stop.process_identity_matches", return_value=None),
                patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
                patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
                patch("praxist.cli.stop._drain_late_run_processes", return_value={}),
                patch("praxist.cli.stop.pid_is_alive", side_effect=lambda pid: pid in alive),
                patch("praxist.cli.stop.os.kill", side_effect=signal_process),
                patch("praxist.cli.stop.time.sleep"),
            ):
                outcome = stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)

        self.assertNotIn((6100, 15), signals)
        self.assertNotIn((6101, 15), signals)
        self.assertIn((6200, 15), signals)
        self.assertEqual(outcome.remaining_pids, [6100, 6101])
        self.assertEqual(outcome.failed_run_ids, [blocked_id])
        self.assertEqual(registry.read_entry(blocked_id).state, registry.STATE_RUNNING)
        self.assertEqual(registry.read_entry(fenced_id).state, registry.STATE_STOPPED)

    def test_stop_all_preserves_unfenced_scheduler_work_after_root_exit(self) -> None:
        from praxist.cli import registry, stop

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            run_dir.mkdir()
            run_id = _write_entry(
                run_id="run_unfenced_dead_root",
                pid=6100,
                run_dir=str(run_dir),
            )
            ps_rows = {
                6101: (1, "00:09", "python -m praxist.run run --task-path /replacement"),
            }

            with (
                patch(
                    "praxist.cli.stop._write_run_shutdown_fence",
                    return_value=False,
                ),
                patch(
                    "praxist.cli.stop._read_identity_bound_ps_snapshot",
                    return_value=(ps_rows, {6101: "proc:6101"}),
                ),
                patch("praxist.cli.stop.process_identity_matches", return_value=None),
                patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
                patch(
                    "praxist.cli.stop._scheduler_owned_state",
                    return_value=({6101}, {6101: {6101: "proc:6101"}}),
                ),
                patch("praxist.cli.stop._run_dir_from_process_environment", return_value=None),
                patch("praxist.cli.stop._process_cwd_is_run_workspace", return_value=False),
                patch("praxist.cli.stop.pid_is_alive", side_effect=lambda pid: pid == 6101),
                patch("praxist.cli.stop.os.kill") as kill,
            ):
                outcome = stop.stop_all(scope=stop.SCOPE_UNION, grace_seconds=0.0)

        kill.assert_not_called()
        self.assertEqual(outcome.remaining_pids, [6101])
        self.assertEqual(outcome.failed_run_ids, [run_id])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_stop_all_reports_unfenced_run_even_when_no_process_is_visible(self) -> None:
        from praxist.cli import registry, stop

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            run_dir.mkdir()
            run_id = _write_entry(
                run_id="run_unfenced_not_visible",
                pid=6100,
                run_dir=str(run_dir),
            )
            with (
                patch(
                    "praxist.cli.stop._write_run_shutdown_fence",
                    return_value=False,
                ),
                patch(
                    "praxist.cli.stop._read_identity_bound_ps_snapshot",
                    return_value=({}, {}),
                ),
                patch("praxist.cli.stop.process_identity_matches", return_value=None),
                patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
                patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
                patch("praxist.cli.stop.pid_is_alive", return_value=False),
            ):
                outcome = stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)

        self.assertEqual(outcome.remaining_pids, [])
        self.assertEqual(outcome.failed_run_ids, [run_id])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_union_stop_skips_ambiguous_ps_scan_when_a_run_is_unfenced(self) -> None:
        from praxist.cli import registry, stop

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            run_dir.mkdir()
            run_id = _write_entry(
                run_id="run_unfenced_without_proc",
                pid=6100,
                run_dir=str(run_dir),
            )
            ps_rows = {
                6200: (1, "00:09", "python -m praxist.run run --task-path /unknown"),
            }
            with (
                patch(
                    "praxist.cli.stop._write_run_shutdown_fence",
                    return_value=False,
                ),
                patch(
                    "praxist.cli.stop._read_identity_bound_ps_snapshot",
                    return_value=(ps_rows, {6200: "ps:stable-start"}),
                ),
                patch("praxist.cli.stop.process_identity_matches", return_value=None),
                patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
                patch("praxist.cli.stop._scheduler_owned_state", return_value=(set(), {})),
                patch("praxist.cli.stop._run_dir_from_process_environment", return_value=None),
                patch("praxist.cli.stop._process_cwd_is_run_workspace", return_value=False),
                patch("praxist.cli.stop.pid_is_alive", side_effect=lambda pid: pid == 6200),
                patch("praxist.cli.stop.os.kill") as kill,
            ):
                outcome = stop.stop_all(scope=stop.SCOPE_UNION, grace_seconds=0.0)

        kill.assert_not_called()
        self.assertEqual(outcome.failed_run_ids, [run_id])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)
        self.assertTrue(any("ps scan was skipped" in item for item in outcome.warnings))

    def test_registry_read_race_is_reported_without_signalling(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(run_id="run_disappeared", pid=6300)
        with (
            patch(
                "praxist.cli.stop.read_entry",
                side_effect=registry.RegistryError("entry disappeared"),
            ),
            patch(
                "praxist.cli.stop._read_identity_bound_ps_snapshot",
                return_value=({}, {}),
            ),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
            patch("praxist.cli.stop.os.kill") as kill,
        ):
            outcome = stop.stop_all(
                scope=stop.SCOPE_REGISTRY,
                grace_seconds=0.0,
            )

        kill.assert_not_called()
        self.assertTrue(
            any(
                run_id in warning and "entry disappeared" in warning for warning in outcome.warnings
            )
        )
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_registry_global_lock_error_is_translated_to_stop_error(self) -> None:
        from praxist.cli import registry, stop

        with (
            patch(
                "praxist.cli.stop.registry_lock",
                side_effect=registry.RegistryError("global lock unavailable"),
            ),
            self.assertRaisesRegex(stop.StopError, "global lock unavailable") as raised,
        ):
            stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)

        self.assertIsInstance(raised.exception.__cause__, registry.RegistryError)


class StopCliEndToEndTest(_StateDirMixin, unittest.TestCase):
    """``praxist stop`` end-to-end through the top-level dispatcher."""

    def _run(self, argv: list[str]) -> tuple[int, str, str]:
        from praxist.cli import main

        stdout, stderr = io.StringIO(), io.StringIO()
        try:
            with redirect_stdout(stdout), redirect_stderr(stderr):
                main(argv)
            code = 0
        except SystemExit as exc:
            code = int(exc.code or 0)
        return code, stdout.getvalue(), stderr.getvalue()

    def test_run_id_and_all_are_mutually_exclusive(self) -> None:
        code, _out, err = self._run(["stop", "some_id", "--all"])
        self.assertEqual(code, 2)
        self.assertIn("mutually exclusive", err)

    def test_neither_run_id_nor_all_is_rejected(self) -> None:
        code, _out, err = self._run(["stop"])
        self.assertEqual(code, 2)
        self.assertIn("expected", err)

    def test_registry_only_and_ps_scan_only_mutex(self) -> None:
        code, _out, err = self._run(["stop", "--all", "--registry-only", "--ps-scan-only"])
        self.assertEqual(code, 2)
        self.assertIn("--registry-only and --ps-scan-only", err)

    def test_non_finite_grace_is_rejected(self) -> None:
        code, _out, err = self._run(["stop", "some_id", "--grace", "nan"])

        self.assertEqual(code, 2)
        self.assertIn("finite non-negative", err)

    def test_stop_run_id_unknown_returns_nonzero(self) -> None:
        code, _out, err = self._run(["stop", "nonexistent"])
        self.assertEqual(code, 1)
        self.assertIn("praxist stop", err)

    def test_epoch_mismatch_returns_nonzero_json_without_changing_state(self) -> None:
        from praxist.cli import registry

        run_id = _write_entry()
        with (
            patch("praxist.cli.stop.entry_process_epoch_matches", return_value=False),
            patch("praxist.cli.stop.os.kill") as kill,
        ):
            code, out, _err = self._run(["stop", run_id, "--json"])

        payload = json.loads(out)
        self.assertEqual(code, 1)
        self.assertEqual(payload["failed_run_ids"], [run_id])
        kill.assert_not_called()
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_stop_run_text_summary_to_stderr(self) -> None:
        """Non-JSON output writes the operator narrative to stderr + matched pids to stdout."""
        run_id = _write_entry()
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
        ):
            code, out, err = self._run(["stop", run_id])
        self.assertEqual(code, 0)
        self.assertIn("praxist stop", err)
        self.assertIn("matched roots", err)
        self.assertIn("5000", out)

    def test_stop_all_text_summary_with_no_matches(self) -> None:
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            code, _out, err = self._run(["stop", "--all"])
        self.assertEqual(code, 0)
        self.assertIn("no matching Praxist runs found", err)

    def test_stop_all_returns_nonzero_when_admission_cannot_close(self) -> None:
        from praxist.cli import stop

        outcome = stop.StopOutcome(
            run_id=None,
            failed_run_ids=["run_admission_open"],
        )
        with patch("praxist.cli.stop.stop_all", return_value=outcome):
            code, _out, err = self._run(["stop", "--all"])

        self.assertEqual(code, 1)
        self.assertIn("admission failed", err)
        self.assertIn("run_admission_open", err)

    def test_stop_returns_nonzero_when_pids_remain_alive(self) -> None:
        """If a pid is still alive after SIGKILL the command exits 1."""
        from praxist.cli import stop

        run_id = _write_entry()
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        # alive throughout — simulates a stubborn root (zombie / EPERM).
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.time.monotonic", side_effect=[0.0, 1.0]),
        ):
            outcome = stop.stop_run(run_id=run_id, grace_seconds=0.0)
        self.assertEqual(outcome.remaining_pids, [5000])

    def test_signal_set_records_permission_errors_and_process_lookup(self) -> None:
        """``_signal_set`` warns on EPERM and silently skips ESRCH."""
        from praxist.cli import stop

        outcome = stop.StopOutcome(run_id="x")

        def fake_kill(pid: int, sig: int) -> None:
            if pid == 100:
                raise ProcessLookupError
            if pid == 200:
                raise PermissionError("nope")

        with patch("praxist.cli.stop.os.kill", side_effect=fake_kill):
            stop._signal_set({100, 200, 300}, 15, outcome)
        # 100: ProcessLookupError → silently skipped, no warning.
        self.assertFalse(any("100" in w for w in outcome.warnings))
        # 200: PermissionError → warning recorded.
        self.assertTrue(any("200" in w for w in outcome.warnings))

    def test_signal_process_groups_handles_lookup_and_permission_errors(self) -> None:
        from praxist.cli import stop

        outcome = stop.StopOutcome(run_id="x")
        attempted: list[int] = []

        def fake_killpg(pgid: int, _sig: int) -> None:
            attempted.append(pgid)
            if pgid == 300:
                raise ProcessLookupError
            if pgid == 200:
                raise PermissionError("denied")

        with patch("praxist.cli.stop.os.killpg", side_effect=fake_killpg):
            stop._signal_process_groups({100, 200, 300}, 15, outcome)

        self.assertEqual(attempted, [300, 200, 100])
        self.assertFalse(any("300" in warning for warning in outcome.warnings))
        self.assertTrue(any("200" in warning for warning in outcome.warnings))

    def test_process_group_liveness_treats_permission_as_alive(self) -> None:
        from praxist.cli import stop

        with patch("praxist.cli.stop.os.killpg", side_effect=ProcessLookupError):
            self.assertFalse(stop._process_group_alive(7000))
        with patch("praxist.cli.stop.os.killpg", side_effect=PermissionError):
            self.assertTrue(stop._process_group_alive(7000))

    def test_validate_registry_root_warns_when_pid_alive_but_absent_from_ps(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(pid=99001)
        ps_rows: dict[int, tuple[int, str, str]] = {}
        outcome = stop.StopOutcome(run_id=run_id)
        with patch("praxist.cli.stop.pid_is_alive", return_value=True):
            root = stop._validate_registry_root(registry.read_entry(run_id), ps_rows, outcome)
        self.assertIsNone(root)
        self.assertTrue(any("absent from ps" in w for w in outcome.warnings))

    def test_validate_registry_root_rejects_recycled_pid_for_different_run_dir(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(
            command=(
                "python",
                "-m",
                "praxist.run",
                "run",
                "--run-dir",
                "/tmp/stop_demo",
            )
        )
        ps_rows = {
            5000: (
                1,
                "00:10",
                "python -m praxist.run run --run-dir /tmp/different_run",
            )
        }
        outcome = stop.StopOutcome(run_id=run_id)

        root = stop._validate_registry_root(registry.read_entry(run_id), ps_rows, outcome)

        self.assertIsNone(root)
        self.assertTrue(any("run directory" in warning for warning in outcome.warnings))

    def test_stop_all_kills_survivors_after_sigterm(self) -> None:
        """SIGTERM-then-SIGKILL escalation in the --all path."""
        import signal as _sig

        from praxist.cli import stop

        _write_entry()
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        killed: list[tuple[int, int]] = []
        kill_phase = {"value": 0}

        def fake_alive(pid: int) -> bool:
            # alive through TERM (phase 0/1), dead after KILL (phase 2).
            return kill_phase["value"] < 2

        def fake_kill(pid: int, sig: int) -> None:
            killed.append((pid, sig))
            if sig == _sig.SIGKILL:
                kill_phase["value"] = 2

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", side_effect=fake_alive),
            patch("praxist.cli.stop.os.kill", side_effect=fake_kill),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.time.monotonic", side_effect=[0.0, 1.0]),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_UNION, grace_seconds=0.05)
        self.assertIn(_sig.SIGKILL, [s for _p, s in killed])
        self.assertEqual(outcome.killed_pids, [5000])

    def test_stop_all_skips_stale_registry_entries(self) -> None:
        """stop_all in registry scope silently skips entries whose pid is gone."""
        from praxist.cli import stop

        _write_entry(run_id="run_alive", pid=5000)
        _write_entry(run_id="run_gone", pid=99002)
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)
        self.assertEqual(outcome.matched_pids, [5000])
        self.assertNotIn(99002, outcome.matched_pids)

    def test_stop_all_preserves_unverified_live_pid_as_remaining(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(pid=5000)
        ps_rows = {5000: (1, "00:10", "/usr/bin/unrelated-process")}
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)

        self.assertEqual(outcome.remaining_pids, [5000])
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_stop_all_includes_scheduler_work_for_dead_registry_root(self) -> None:
        from praxist.cli import stop

        _write_entry(run_id="run_dead_root", pid=99002)
        ps_rows = {6001: (1, "00:02", "python evaluator.py")}
        alive = {6001}
        group_alive = {6001}
        group_signals: list[tuple[int, int]] = []
        pid_signals: list[tuple[int, int]] = []

        def fake_alive(pid: int) -> bool:
            return pid in alive

        def fake_kill(pid: int, sig: int) -> None:
            pid_signals.append((pid, sig))
            alive.discard(pid)

        def fake_killpg(pgid: int, sig: int) -> None:
            if sig == 0:
                if pgid not in group_alive:
                    raise ProcessLookupError
                return
            group_signals.append((pgid, sig))
            group_alive.discard(pgid)

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", side_effect=fake_alive),
            patch("praxist.cli.stop.os.kill", side_effect=fake_kill),
            patch("praxist.cli.stop.os.killpg", side_effect=fake_killpg),
            patch("praxist.cli.stop.os.getpgid", return_value=6001),
            patch("praxist.cli.stop.process_start_token", return_value="proc:123"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
            patch(
                "praxist.cli.stop._scheduler_owned_state",
                return_value=({6001}, {6001: {6001: "proc:123"}}),
            ),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_REGISTRY, grace_seconds=0.0)

        self.assertEqual(outcome.matched_pids, [])
        self.assertEqual(outcome.descendant_pids, [6001])
        self.assertIn((6001, 15), pid_signals)
        self.assertNotIn((6001, 15), group_signals)

    def test_stop_all_excludes_self_ancestor_pids_from_ps_scan(self) -> None:
        from praxist.cli import stop

        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
            6000: (1, "00:01", "python -m praxist.run run --task-path /other"),
        }
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value={6000}),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_PS, grace_seconds=0.0)
        self.assertEqual(outcome.matched_pids, [5000])

    def test_gather_targets_handles_cycles_gracefully(self) -> None:
        """A pathological ps table with a parent-child cycle must not loop forever."""
        from praxist.cli import stop

        ps_rows: dict[int, tuple[int, str, str]] = {
            100: (200, "00:01", "cmd_a"),
            200: (100, "00:01", "cmd_b"),  # cycle: 100<->200
        }
        seen = stop._gather_targets(root_pids={100}, ps_rows=ps_rows)
        self.assertEqual(seen, {100, 200})

    def test_gather_targets_dedupes_duplicate_stack_pushes(self) -> None:
        """The seen-check at pop time defends against duplicate pushes.

        Construct a ps table where root B's child is root A — both roots
        push A onto the stack, and the pop loop must reject the second
        copy without re-processing it.
        """
        from praxist.cli import stop

        ps_rows: dict[int, tuple[int, str, str]] = {
            100: (1, "00:01", "cmd_a"),
            200: (1, "00:01", "cmd_b"),
            300: (200, "00:01", "cmd_a_child_of_b"),
        }
        seen = stop._gather_targets(root_pids={100, 200, 300}, ps_rows=ps_rows)
        self.assertEqual(seen, {100, 200, 300})

    def test_await_exit_sleeps_until_targets_clear(self) -> None:
        from praxist.cli import stop

        outcome = stop.StopOutcome(run_id="x")
        alive_calls = {"n": 0}

        def alive(pid: int) -> bool:
            alive_calls["n"] += 1
            return alive_calls["n"] < 3  # stays alive for 2 polls, then dies.

        with (
            patch("praxist.cli.stop.pid_is_alive", side_effect=alive),
            patch("praxist.cli.stop.time.sleep") as sleep_mock,
            patch(
                "praxist.cli.stop.time.monotonic",
                side_effect=[0.0, 0.1, 0.2, 0.3, 1.0],
            ),
        ):
            stop._await_exit({5000}, grace_seconds=0.5, outcome=outcome)
        self.assertGreaterEqual(sleep_mock.call_count, 1)

    def test_stop_all_warning_when_registry_update_fails(self) -> None:
        from praxist.cli import stop

        _write_entry()
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
            patch(
                "praxist.cli.stop.update_state",
                side_effect=OSError("disk full"),
            ),
        ):
            outcome = stop.stop_all(scope=stop.SCOPE_UNION, grace_seconds=0.0)
        self.assertTrue(any("could not mark" in w for w in outcome.warnings))

    def test_stop_exit_code_one_when_pids_remain(self) -> None:
        run_id = _write_entry()
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
            patch("praxist.cli.stop.time.monotonic", side_effect=[0.0, 100.0]),
        ):
            code, _out, err = self._run(["stop", run_id])
        self.assertEqual(code, 1)
        self.assertIn("still alive", err)

    def test_stop_run_dry_run_summary_to_stderr(self) -> None:
        run_id = _write_entry()
        ps_rows = {5000: (1, "00:10", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill") as kill_mock,
            patch("praxist.cli.stop.time.sleep"),
        ):
            code, _out, err = self._run(["stop", run_id, "--dry-run"])
        self.assertEqual(code, 0)
        self.assertIn("(dry-run", err)
        kill_mock.assert_not_called()

    def test_stop_run_with_warning_renders_in_summary(self) -> None:
        run_id = _write_entry()
        # ps row has different cmdline → TOCTOU warning + matched_pids empty.
        ps_rows = {5000: (1, "00:01", "/bin/bash")}
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.time.sleep"),
        ):
            code, _out, err = self._run(["stop", run_id])
        self.assertEqual(code, 1)
        self.assertIn("warning", err)
        self.assertIn("TOCTOU", err)

    def test_stop_all_dry_run_with_json(self) -> None:
        _write_entry()
        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
        }
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
            patch("praxist.cli.stop.os.kill"),
            patch("praxist.cli.stop.self_ancestor_pids", return_value=set()),
        ):
            code, out, _err = self._run(["stop", "--all", "--dry-run", "--json"])
        self.assertEqual(code, 0)
        payload = json.loads(out)
        self.assertTrue(payload["dry_run"])
        self.assertIn(5000, payload["matched_pids"])
        self.assertEqual(payload["monitor_sessions"], [])
        self.assertEqual(payload["monitor_stopped_sessions"], [])


class GcStaleEntriesTest(_StateDirMixin, unittest.TestCase):
    """``praxist stop --gc`` cleanup of stale registry entries (#166)."""

    def test_alive_entry_with_matching_prefix_is_kept(self) -> None:
        """Live PID + matching command prefix → entry stays, no warning."""
        from praxist.cli import registry, stop

        run_id = _write_entry()
        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
        }
        with patch("praxist.cli.stop.read_ps_table", return_value=ps_rows):
            outcome = stop.gc_stale_entries()
        self.assertEqual(outcome.removed_run_ids, [])
        self.assertEqual(outcome.kept_run_ids, [run_id])
        self.assertEqual(outcome.warnings, [])
        # File is still on disk.
        self.assertEqual(registry.read_entry(run_id).pid, 5000)

    def test_dead_pid_removes_entry(self) -> None:
        """PID is not in ps and not alive → entry is unlinked."""
        from praxist.cli import registry, stop

        run_id = _write_entry()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
        ):
            outcome = stop.gc_stale_entries()
        self.assertEqual(outcome.removed_run_ids, [run_id])
        self.assertEqual(outcome.kept_run_ids, [])
        # Entry file is gone.
        with self.assertRaises(registry.RegistryError):
            registry.read_entry(run_id)

    def test_prior_boot_entry_is_removed_without_pid_probe(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry()
        with (
            patch("praxist.cli.stop.entry_process_epoch_matches", return_value=False),
            patch("praxist.cli.stop.pid_is_alive") as alive,
        ):
            outcome = stop.gc_stale_entries()

        alive.assert_not_called()
        self.assertEqual(outcome.removed_run_ids, [run_id])
        with self.assertRaises(registry.RegistryError):
            registry.read_entry(run_id)

    def test_recycled_pid_with_foreign_command_removes_entry(self) -> None:
        """PID is alive but the command line no longer starts with the
        prefix we recorded → entry is treated as stale."""
        from praxist.cli import registry, stop

        run_id = _write_entry()
        ps_rows = {
            5000: (1, "00:10", "vim some_file"),  # cmdline drifted; not Praxist
        }
        with patch("praxist.cli.stop.read_ps_table", return_value=ps_rows):
            outcome = stop.gc_stale_entries()
        self.assertEqual(outcome.removed_run_ids, [run_id])
        self.assertEqual(outcome.kept_run_ids, [])
        with self.assertRaises(registry.RegistryError):
            registry.read_entry(run_id)

    def test_recycled_pid_with_matching_command_but_new_start_token_is_removed(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry()
        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
        }
        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.process_identity_matches", return_value=False),
        ):
            outcome = stop.gc_stale_entries()

        self.assertEqual(outcome.removed_run_ids, [run_id])
        with self.assertRaises(registry.RegistryError):
            registry.read_entry(run_id)

    def test_matching_start_token_keeps_entry_when_command_changes(self) -> None:
        from praxist.cli import stop

        run_id = _write_entry(extra={"process_start_token": "proc:live"})
        with (
            patch(
                "praxist.cli.stop.read_ps_table",
                return_value={5000: (1, "00:01", "renamed-controller")},
            ),
            patch("praxist.cli.stop.process_identity_matches", return_value=True),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
        ):
            outcome = stop.gc_stale_entries()

        self.assertEqual(outcome.kept_run_ids, [run_id])
        self.assertEqual(outcome.removed_run_ids, [])

    def test_recycled_pid_with_same_prefix_but_other_run_dir_removes_entry(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(
            command=(
                "python",
                "-m",
                "praxist.run",
                "run",
                "--run-dir",
                "/tmp/stop_demo",
            )
        )
        ps_rows = {
            5000: (
                1,
                "00:10",
                "python -m praxist.run run --run-dir /tmp/different_run",
            )
        }

        with patch("praxist.cli.stop.read_ps_table", return_value=ps_rows):
            outcome = stop.gc_stale_entries()

        self.assertEqual(outcome.removed_run_ids, [run_id])
        with self.assertRaises(registry.RegistryError):
            registry.read_entry(run_id)

    def test_dry_run_lists_without_unlinking(self) -> None:
        """``--dry-run`` reports the would-be-removed ids and leaves the
        registry intact, so the operator can preview before destruction."""
        from praxist.cli import registry, stop

        run_id = _write_entry()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
        ):
            outcome = stop.gc_stale_entries(dry_run=True)
        self.assertEqual(outcome.removed_run_ids, [run_id])
        self.assertTrue(outcome.dry_run)
        # Entry survives the dry-run.
        self.assertEqual(registry.read_entry(run_id).pid, 5000)

    def test_alive_pid_not_in_ps_is_kept_with_warning(self) -> None:
        """PID is alive but ``ps`` doesn't list it (different user /
        container / permission boundary). GC refuses to delete — we
        can't verify ownership, and the file might belong to a run
        we just can't see."""
        from praxist.cli import stop

        run_id = _write_entry()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.pid_is_alive", return_value=True),
        ):
            outcome = stop.gc_stale_entries()
        self.assertEqual(outcome.removed_run_ids, [])
        self.assertEqual(outcome.kept_run_ids, [run_id])
        self.assertEqual(len(outcome.warnings), 1)
        self.assertIn("cannot verify ownership", outcome.warnings[0])

    def test_mixed_registry_partitions_correctly(self) -> None:
        """Three entries — one alive-matching, one dead, one recycled —
        each routes through the correct branch. Live one stays; the
        other two are removed."""
        from praxist.cli import registry, stop

        live = _write_entry(run_id="run_live", pid=5000)
        dead = _write_entry(run_id="run_dead", pid=6000)
        recycled = _write_entry(run_id="run_recycled", pid=7000)
        ps_rows = {
            5000: (1, "00:10", "python -m praxist.run run --task-path /t"),
            7000: (1, "00:05", "vim some_other_file"),
        }

        def _alive(pid: int) -> bool:
            return pid != 6000  # dead

        with (
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
            patch("praxist.cli.stop.pid_is_alive", side_effect=_alive),
        ):
            outcome = stop.gc_stale_entries()
        self.assertEqual(outcome.removed_run_ids, sorted([dead, recycled]))
        self.assertEqual(outcome.kept_run_ids, [live])
        self.assertEqual(registry.read_entry(live).pid, 5000)

    def test_empty_registry_is_a_quiet_no_op(self) -> None:
        from praxist.cli import stop

        with patch("praxist.cli.stop.read_ps_table", return_value={}):
            outcome = stop.gc_stale_entries()
        self.assertEqual(outcome.removed_run_ids, [])
        self.assertEqual(outcome.kept_run_ids, [])
        self.assertEqual(outcome.warnings, [])

    def test_gc_rereads_entry_after_acquiring_lifecycle_lock(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(pid=5000)
        replacement = registry.RegistryEntry(**_entry_kwargs(pid=6000))

        @contextmanager
        def replace_before_lock_body(_run_id: str):
            registry.write_entry(replacement)
            yield

        ps_rows = {6000: (1, "00:01", "python -m praxist.run run --task-path /t")}
        with (
            patch("praxist.cli.stop.entry_lock", side_effect=replace_before_lock_body),
            patch("praxist.cli.stop.read_ps_table", return_value=ps_rows),
        ):
            outcome = stop.gc_stale_entries()

        self.assertEqual(outcome.kept_run_ids, [run_id])
        self.assertEqual(outcome.removed_run_ids, [])
        self.assertEqual(registry.read_entry(run_id).pid, 6000)

    def test_gc_reports_registry_lock_race_and_keeps_entry(self) -> None:
        from praxist.cli import registry, stop

        run_id = _write_entry(run_id="run_gc_locked")
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch(
                "praxist.cli.stop.entry_lock",
                side_effect=registry.RegistryError("entry lock changed"),
            ),
        ):
            outcome = stop.gc_stale_entries()

        self.assertEqual(outcome.removed_run_ids, [])
        self.assertTrue(
            any(
                run_id in warning and "entry lock changed" in warning
                for warning in outcome.warnings
            )
        )
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)

    def test_gc_reports_already_gone_and_unlink_errors(self) -> None:
        from praxist.cli import registry, stop

        already_gone_id = _write_entry(run_id="run_gc_already_gone", pid=7100)
        unlink_error_id = _write_entry(run_id="run_gc_unlink_error", pid=7200)

        def remove_entry(run_id: str) -> bool:
            if run_id == already_gone_id:
                return False
            raise OSError("permission denied")

        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
            patch("praxist.cli.stop.remove_entry", side_effect=remove_entry),
        ):
            outcome = stop.gc_stale_entries()

        self.assertEqual(outcome.removed_run_ids, [])
        self.assertTrue(any("already gone" in warning for warning in outcome.warnings))
        self.assertTrue(any("permission denied" in warning for warning in outcome.warnings))
        self.assertEqual(registry.read_entry(already_gone_id).pid, 7100)
        self.assertEqual(registry.read_entry(unlink_error_id).pid, 7200)


class GcCliDispatchTest(_StateDirMixin, unittest.TestCase):
    """``praxist stop --gc`` end-to-end through the top-level dispatcher."""

    def _run(self, argv: list[str]) -> tuple[int, str, str]:
        from praxist.cli import main

        stdout, stderr = io.StringIO(), io.StringIO()
        try:
            with redirect_stdout(stdout), redirect_stderr(stderr):
                main(argv)
            code = 0
        except SystemExit as exc:
            code = int(exc.code or 0)
        return code, stdout.getvalue(), stderr.getvalue()

    def test_gc_alone_runs_through_dispatcher(self) -> None:
        from praxist.cli import registry

        run_id = _write_entry()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
        ):
            code, out, err = self._run(["stop", "--gc"])
        self.assertEqual(code, 0)
        self.assertIn(run_id, out)  # stdout: one run_id per line
        self.assertIn("removed", err)
        with self.assertRaises(registry.RegistryError):
            registry.read_entry(run_id)

    def test_gc_with_dry_run_does_not_unlink(self) -> None:
        from praxist.cli import registry

        run_id = _write_entry()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
        ):
            code, out, err = self._run(["stop", "--gc", "--dry-run"])
        self.assertEqual(code, 0)
        self.assertIn(run_id, out)
        self.assertIn("dry-run", err)
        # Entry survives dry-run.
        self.assertEqual(registry.read_entry(run_id).pid, 5000)

    def test_gc_with_run_id_is_rejected(self) -> None:
        code, _out, err = self._run(["stop", "some_id", "--gc"])
        self.assertEqual(code, 2)
        self.assertIn("--gc cannot be combined", err)

    def test_gc_with_all_is_rejected(self) -> None:
        code, _out, err = self._run(["stop", "--all", "--gc"])
        self.assertEqual(code, 2)
        self.assertIn("--gc cannot be combined", err)

    def test_gc_with_json_emits_structured_payload(self) -> None:
        _write_entry()
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.pid_is_alive", return_value=False),
        ):
            code, out, _err = self._run(["stop", "--gc", "--json"])
        self.assertEqual(code, 0)
        payload = json.loads(out)
        self.assertFalse(payload["dry_run"])
        self.assertEqual(payload["removed_run_ids"], ["run_stop_demo"])
        self.assertEqual(payload["kept_run_ids"], [])

    def test_gc_keeps_remote_entry_and_summarizes_reason(self) -> None:
        from praxist.cli import registry

        run_id = _write_entry(extra={"hostname": "remote.example"})
        with (
            patch("praxist.cli.stop.read_ps_table", return_value={}),
            patch("praxist.cli.stop.entry_is_local", return_value=False),
        ):
            code, out, err = self._run(["stop", "--gc"])

        self.assertEqual(code, 0)
        self.assertEqual(out, "")
        self.assertIn("no stale entries found", err)
        self.assertIn("kept   : 1 live entry", err)
        self.assertIn("remote-host registry entry kept", err)
        self.assertEqual(registry.read_entry(run_id).state, registry.STATE_RUNNING)


if __name__ == "__main__":  # pragma: no cover
    unittest.main()