Skip to content

Autonomous research system for measurable, computer-executable research.

Python
332 KB
Raw Blame
from __future__ import annotations

import contextlib
import json
import math
import os
import signal
import socket
import subprocess
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch

from praxist.plugins.workflow_stages.research_loop.backend import protected_pids
from praxist.plugins.workflow_stages.research_loop.backend.experiment_process import (
    process_group_alive,
)
from praxist.plugins.workflow_stages.research_loop.backend.experiment_scheduler import (
    ExperimentSchedulerService,
    _ActiveJob,
    _RequestHandler,
    _SupplyLease,
)
from praxist.plugins.workflow_stages.research_loop.backend.experiment_scheduler_client import (
    ExperimentRejected,
    SchedulerUnavailable,
    freeze_generation,
    rebase_recovered_task_context,
    submit_and_wait,
    task_runtime_context_changed,
)
from praxist.plugins.workflow_stages.research_loop.backend.resource_scheduler import (
    Allocation,
    GPUDevice,
    GPUProcess,
    HostAllocationRegistry,
    HostSnapshot,
    ResourceAllocator,
    SchedulerSettings,
)


class _GPUAllocator:
    def __init__(self, uuid: str, limit: int = 2) -> None:
        self.uuid = uuid
        self.concurrency_limit = limit
        self.snapshot = HostSnapshot(8, 10, 10, 0)
        self.released: list[str] = []
        self.active: set[str] = set()

    def refresh(self, *, queued: bool) -> HostSnapshot:
        return self.snapshot

    def reserve(self, **kwargs) -> Allocation | None:
        if len(self.active) >= self.concurrency_limit:
            return None
        self.active.add(kwargs["allocation_id"])
        return Allocation(
            allocation_id=kwargs["allocation_id"],
            run_id=kwargs["run_id"],
            pid=kwargs["pid"],
            pgid=kwargs["pgid"],
            profile=kwargs["profile"].name,
            gpu_uuids=(self.uuid,) if self.uuid else (),
            gpu_memory_mb=1024,
            gpu_utilization_pct=20,
            started_at=time.time(),
        )

    def bind_process(self, allocation_id: str, *, pid: int, pgid: int) -> bool:
        return allocation_id in self.active

    def recover_allocation(self, **kwargs) -> Allocation:
        self.active.add(kwargs["allocation_id"])
        return Allocation(
            allocation_id=kwargs["allocation_id"],
            run_id=kwargs["run_id"],
            pid=kwargs["pid"],
            pgid=kwargs["pgid"],
            profile=kwargs["profile"].name,
            gpu_uuids=tuple(kwargs["gpu_uuids"]),
            gpu_memory_mb=1024,
            gpu_utilization_pct=20,
            started_at=time.time(),
        )

    def release(self, allocation_id: str) -> None:
        self.released.append(allocation_id)
        self.active.discard(allocation_id)

    def describe_allocation_activity(self, allocation: Allocation) -> dict[str, object]:
        return {"state": "gpu_compute_active" if allocation.gpu_uuids else "non_gpu_allocation"}


class _StaticObserver:
    def snapshot(self) -> HostSnapshot:
        return HostSnapshot(8, 10, 10, 0)


class _BlockedAllocator(_GPUAllocator):
    def reserve(self, **kwargs) -> None:
        return None


class _BindFailAllocator(_GPUAllocator):
    def bind_process(self, allocation_id: str, *, pid: int, pgid: int) -> None:
        raise OSError("binding ledger unavailable")


class _BindingMissingAllocator(_GPUAllocator):
    def bind_process(self, allocation_id: str, *, pid: int, pgid: int) -> bool:
        self.active.discard(allocation_id)
        return False


class _SupplyAllocator(_GPUAllocator):
    def __init__(self, slots: tuple[tuple[str, ...], ...]) -> None:
        super().__init__("", limit=max(1, len(slots)))
        self.slots = slots
        self.claims: dict[str, tuple[str, ...]] = {}

    def claim_supply(self, *, lease_id: str, run_id: str, expires_at: float) -> tuple[str, ...]:
        del run_id, expires_at
        if len(self.claims) >= len(self.slots):
            return ()
        profiles = self.slots[len(self.claims)]
        self.claims[lease_id] = profiles
        return profiles

    def supply_claim_valid(self, lease_id: str) -> bool:
        return lease_id in self.claims

    def release(self, allocation_id: str) -> None:
        self.claims.pop(allocation_id, None)
        super().release(allocation_id)


def _settings(*, maximum: int = 2, initial: int | None = None) -> SchedulerSettings:
    return SchedulerSettings.from_dict(
        {
            "mode": "central",
            "initial_concurrent_experiments": initial or maximum,
            "max_concurrent_experiments": maximum,
            "adjustment_interval_seconds": 2,
            "profiles": {
                "cpu": {"accelerator": "cpu"},
                "gpu": {
                    "accelerator": "gpu",
                    "gpu_memory_gb": 20,
                    "gpu_utilization_pct": 40,
                },
            },
            "default_profile": "cpu",
        }
    )


def _write_active_attempt(run_dir: Path, attempt_id: str) -> Path:
    attempt_dir = run_dir / "resource_scheduler" / "attempts" / attempt_id
    attempt_dir.mkdir(parents=True)
    (attempt_dir / "READY.json").write_text(
        json.dumps(
            {
                "pid": os.getpid(),
                "pgid": os.getpgrp(),
                "attempt_id": attempt_id,
            }
        ),
        encoding="utf-8",
    )
    (attempt_dir / "GO.json").write_text('{"go":true}\n', encoding="utf-8")
    return attempt_dir


class ExperimentSchedulerTest(unittest.TestCase):
    def test_rpc_handler_ignores_client_disconnect_while_writing_response(self) -> None:
        handler = object.__new__(_RequestHandler)
        handler.server = MagicMock()
        handler.server.service.handle_request.return_value = {"ok": True}
        handler.rfile = MagicMock()
        handler.rfile.readline.return_value = b'{"action":"ping"}\n'
        handler.wfile = MagicMock()
        handler.wfile.write.side_effect = BrokenPipeError("client exited")

        handler.handle()

        handler.server.service.handle_request.assert_called_once_with({"action": "ping"})

    def test_unavailable_accelerator_rejects_only_gpu_work(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _GPUAllocator("", limit=2)
            allocator.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                accelerator_probe_state="unavailable",
                accelerator_probe_reason="nvidia-smi is not installed",
            )
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=2),
                allocator=allocator,
            )
            cpu = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "cpu-still-works",
                    "profile": "cpu",
                }
            )
            self.assertEqual(cpu.state, "queued")
            with self.assertRaisesRegex(ExperimentRejected, "accelerator_unavailable"):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer1",
                        "generation_id": 0,
                        "experiment_id": "gpu-cannot-run",
                        "profile": "gpu",
                    }
                )

    def test_transient_accelerator_probe_has_bounded_queue_wait(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _GPUAllocator("", limit=2)
            allocator.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                accelerator_probe_state="unknown",
                accelerator_probe_reason="driver query timed out",
            )
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=2),
                allocator=allocator,
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "gpu-probe-recovery-window",
                    "profile": "gpu",
                }
            )
            assert service._accelerator_probe_unknown_since is not None
            service._accelerator_probe_unknown_since -= 121
            service._launch_ready_jobs()
            self.assertEqual(job.state, "rejected")
            self.assertIn("accelerator_probe_unknown", job.error)
            scheduler_status = service.status()
            self.assertEqual(scheduler_status["accelerator_probe"]["state"], "unknown")

    def test_same_experiment_can_be_resubmitted_after_accelerator_probe_recovers(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _BlockedAllocator("", limit=1)
            allocator.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                accelerator_probe_state="unknown",
                accelerator_probe_reason="driver query timed out",
            )
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            request = {
                "command": [sys.executable, "-c", "pass"],
                "peer_id": "gen0_peer0",
                "generation_id": 0,
                "experiment_id": "gpu-probe-retry",
                "profile": "gpu",
            }
            rejected = service.submit(request)
            assert service._accelerator_probe_unknown_since is not None
            service._accelerator_probe_unknown_since -= 121
            service._launch_ready_jobs()
            self.assertEqual(rejected.state, "rejected")

            allocator.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                gpus=(GPUDevice(0, "GPU-a", 40 * 1024, 39 * 1024, 100),),
                accelerator_probe_state="available",
            )
            retried = service.submit(request)

            self.assertNotEqual(retried.job_id, rejected.job_id)
            self.assertEqual(retried.state, "queued")
            self.assertEqual(service._semantic_jobs[retried.semantic_key], retried.job_id)

    def test_probe_rejection_remains_resubmittable_after_scheduler_restart(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            unavailable = _BlockedAllocator("", limit=1)
            unavailable.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                accelerator_probe_state="unknown",
                accelerator_probe_reason="driver query timed out",
            )
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=unavailable,
            )
            request = {
                "command": [sys.executable, "-c", "pass"],
                "peer_id": "gen0_peer0",
                "generation_id": 0,
                "experiment_id": "gpu-probe-restart-retry",
                "profile": "gpu",
            }
            rejected = first.submit(request)
            assert first._accelerator_probe_unknown_since is not None
            first._accelerator_probe_unknown_since -= 121
            first._launch_ready_jobs()
            self.assertEqual(rejected.state, "rejected")

            available = _BlockedAllocator("", limit=1)
            available.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                gpus=(GPUDevice(0, "GPU-a", 40 * 1024, 39 * 1024, 100),),
                accelerator_probe_state="available",
            )
            resumed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=available,
            )
            resumed._recover_terminal_events()
            retried = resumed.submit(request)

            self.assertNotEqual(retried.job_id, rejected.job_id)
            self.assertEqual(retried.state, "queued")
            self.assertEqual(resumed._semantic_jobs[retried.semantic_key], retried.job_id)

    def test_old_gpu_job_survives_first_transient_probe_unknown_snapshot(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _BlockedAllocator("", limit=1)
            allocator.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                gpus=(GPUDevice(0, "GPU-a", 40 * 1024, 39 * 1024, 100),),
                accelerator_probe_state="available",
            )
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "old-job-first-probe-gap",
                    "profile": "gpu",
                }
            )
            job.submitted_at -= 3600
            allocator.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                accelerator_probe_state="unknown",
                accelerator_probe_reason="single inventory timeout",
            )

            service._launch_ready_jobs()

            self.assertEqual(job.state, "queued")
            self.assertIn(job.job_id, service._queue)
            self.assertIsNotNone(service._accelerator_probe_unknown_since)

    def test_structurally_impossible_gpu_profiles_are_rejected_without_queueing(self) -> None:
        gpu = GPUDevice(0, "GPU-a", 40 * 1024, 0, 0)
        for label, profile, expected in (
            (
                "count",
                {
                    "accelerator": "gpu",
                    "gpu_count": 2,
                    "gpu_memory_gb": 10,
                    "gpu_utilization_pct": 20,
                },
                "requests 2 GPUs",
            ),
            (
                "memory",
                {
                    "accelerator": "gpu",
                    "gpu_count": 1,
                    "gpu_memory_gb": 80,
                    "gpu_utilization_pct": 20,
                },
                "requests 80 GiB",
            ),
        ):
            with self.subTest(label=label), tempfile.TemporaryDirectory() as td:
                settings = SchedulerSettings.from_dict(
                    {
                        "mode": "central",
                        "initial_concurrent_experiments": 1,
                        "max_concurrent_experiments": 1,
                        "profiles": {"gpu": profile},
                        "default_profile": "gpu",
                    }
                )
                allocator = _BlockedAllocator("", limit=1)
                allocator.snapshot = HostSnapshot(8, 10, 10, 0, gpus=(gpu,))
                service = ExperimentSchedulerService(
                    run_dir=Path(td) / "run",
                    settings=settings,
                    allocator=allocator,
                )
                with self.assertRaisesRegex(
                    ExperimentRejected,
                    f"accelerator_profile_unsatisfied: .*{expected}",
                ):
                    service.submit(
                        {
                            "command": [sys.executable, "-c", "pass"],
                            "peer_id": "gen0_peer0",
                            "generation_id": 0,
                            "experiment_id": f"impossible-{label}",
                            "profile": "gpu",
                        }
                    )

    def test_temporarily_busy_compatible_gpu_profile_remains_queued(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _BlockedAllocator("", limit=1)
            allocator.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                gpus=(GPUDevice(0, "GPU-a", 40 * 1024, 39 * 1024, 100),),
                accelerator_probe_state="available",
            )
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "temporarily-busy-gpu",
                    "profile": "gpu",
                }
            )
            job.submitted_at -= 3600
            service._launch_ready_jobs()

            self.assertEqual(job.state, "queued")
            self.assertIn(job.job_id, service._queue)

    def test_existing_gpu_semantic_job_survives_later_probe_outage(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _GPUAllocator("GPU-a", limit=1)
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            request = {
                "command": [sys.executable, "-c", "pass"],
                "peer_id": "gen0_peer0",
                "generation_id": 0,
                "experiment_id": "stable-semantic-job",
                "profile": "gpu",
            }
            existing = service.submit(request)
            allocator.snapshot = HostSnapshot(
                8,
                10,
                10,
                0,
                accelerator_probe_state="unavailable",
                accelerator_probe_reason="temporary inventory loss",
            )

            duplicate = service.submit(request)

            self.assertIs(duplicate, existing)
            self.assertEqual(existing.state, "queued")

    def test_same_peer_job_waits_in_queue_until_active_job_drains(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
                max_parallel_runs_per_peer=1,
            )
            first = service.submit(
                {
                    "command": [sys.executable, "-c", "import time; time.sleep(0.8)"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "first",
                    "profile": "cpu",
                }
            )
            second = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "second",
                    "profile": "cpu",
                }
            )
            third = service.submit(
                {
                    "command": [sys.executable, "-c", "import time; time.sleep(0.6)"],
                    "peer_id": "gen0_peer1",
                    "generation_id": 0,
                    "experiment_id": "other-peer",
                    "profile": "cpu",
                }
            )
            service.start()
            try:
                deadline = time.time() + 3
                while (
                    not {first.job_id, third.job_id} <= service._active.keys()
                    and time.time() < deadline
                ):
                    time.sleep(0.02)
                status = service.status()
                self.assertEqual(status["running"], 2)
                self.assertEqual(status["queued"], 1)
                self.assertEqual(status["peer_capacity_blocked"], 1)
                self.assertEqual(second.attempts, 0)
                queued = next(job for job in status["jobs"] if job["state"] == "queued")
                self.assertEqual(queued["queue_blocked_reason"], "per_peer_capacity")
                self.assertEqual(
                    status["running_activity"]["by_resource_phase"],
                    {"non_gpu_allocation": 2},
                )
                running = next(job for job in status["jobs"] if job["state"] == "running")
                self.assertEqual(running["resource_activity"]["state"], "non_gpu_allocation")
                self.assertEqual(service.wait(second.job_id, 5)["job"]["state"], "completed")
            finally:
                service.stop()
            events = (service.state_dir / "events.jsonl").read_text(encoding="utf-8")
            self.assertNotIn("ProtectedPidCapacityError", events)

    def test_durable_command_references_environment_secrets_without_plaintext(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            with patch.dict(os.environ, {"SERVICE_TOKEN": "opaque-secret-value"}, clear=False):
                job = service.submit(
                    {
                        "command": [
                            sys.executable,
                            "task.py",
                            "--auth=opaque-secret-value",
                        ],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "secret-reference",
                        "profile": "cpu",
                        "environment": dict(os.environ),
                    }
                )
                event_text = (service.state_dir / "events.jsonl").read_text(encoding="utf-8")
                self.assertNotIn("opaque-secret-value", event_text)
                self.assertIn("__PRAXIST_ENV_REF_SERVICE_TOKEN__", event_text)

                recovered = ExperimentSchedulerService(
                    run_dir=service.run_dir,
                    settings=_settings(maximum=1),
                    allocator=_GPUAllocator(""),
                )
                recovered._recover_terminal_events()
                self.assertEqual(
                    recovered._jobs[job.job_id].command[-1],
                    "--auth=opaque-secret-value",
                )

    def test_recovered_queued_task_reapplies_task_runtime_boundary(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td).resolve()
            task_root = root / "task"
            evaluator = task_root / "evaluations" / "run.py"
            task_python = task_root / ".venv" / "bin" / "python"
            evaluator.parent.mkdir(parents=True)
            task_python.parent.mkdir(parents=True)
            evaluator.write_text("print('ok')\n", encoding="utf-8")
            task_python.write_text("", encoding="utf-8")
            service = ExperimentSchedulerService(
                run_dir=root / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            event = {
                "command": ["python3.8", "evaluations/run.py"],
                "environment_values": {
                    "PRAXIST_TASK_PROJECT_PATH": str(task_root),
                    "PRAXIST_TASK_PYTHON": str(task_python),
                    "PYTHONPATH": "/runner/python313",
                    "PYTHONHOME": "/runner",
                    "PATH": "/usr/bin",
                },
                "generation_id": 0,
                "peer_id": "gen0_peer0",
                "experiment_id": "recovered-task-boundary",
                "profile": "cpu",
                "cwd": str(root / "run"),
            }

            recovered = service._job_from_event("job", event, state="queued")

        self.assertEqual(recovered.command[0], str(task_python))
        self.assertEqual(recovered.command[1], str(evaluator.resolve()))
        self.assertNotIn("PYTHONPATH", recovered.environment)
        self.assertNotIn("PYTHONHOME", recovered.environment)
        self.assertEqual(
            recovered.environment["PATH"].split(os.pathsep)[0],
            str(task_python.parent),
        )

    def test_recovered_job_resolves_relative_cwd_against_task_root(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            task_root = root / "task"
            work_dir = task_root / "work"
            work_dir.mkdir(parents=True)
            service = ExperimentSchedulerService(
                run_dir=root / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            event = {
                "command": [sys.executable, "-c", "pass"],
                "environment_values": {"PRAXIST_TASK_PROJECT_PATH": str(task_root)},
                "generation_id": 0,
                "peer_id": "gen0_peer0",
                "experiment_id": "relative-recovery-cwd",
                "profile": "cpu",
                "cwd": "work",
            }

            recovered = service._job_from_event("job", event, state="queued")

        self.assertEqual(recovered.cwd, str(work_dir.resolve()))

    def test_recovered_job_rebases_task_owned_paths_after_checkout_moves(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td).resolve()
            old_root = root / "old" / "task"
            new_root = root / "new" / "task"
            old_workspace = root / "old"
            new_workspace = root / "new"
            new_evaluator = new_root / "evaluations" / "run.py"
            new_python = new_root / ".venv" / "bin" / "python"
            new_work = new_root / "work"
            new_evaluator.parent.mkdir(parents=True)
            new_python.parent.mkdir(parents=True)
            new_work.mkdir(parents=True)
            new_evaluator.write_text("print('ok')\n", encoding="utf-8")
            new_python.write_text("", encoding="utf-8")
            service = ExperimentSchedulerService(
                run_dir=root / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            old_python = old_root / ".venv" / "bin" / "python"
            event = {
                "command": [str(old_python), str(old_root / "evaluations" / "run.py")],
                "environment_values": {
                    "PRAXIST_TASK_PROJECT_PATH": str(old_root),
                    "PRAXIST_WORKSPACE_ROOT": str(old_workspace),
                    "PRAXIST_DATASETS_DIR": str(old_workspace / "datasets"),
                    "PRAXIST_TASK_RUNTIME_ENV_KEYS": "TASK_DATA_ALIAS",
                    "TASK_DATA_ALIAS": str(old_workspace / "task_data"),
                    "EXTERNAL_DATA": "/datasets/shared",
                    "PRAXIST_TASK_PYTHON": str(old_python),
                    "PRAXIST_TASK_VENV": str(old_root / ".venv"),
                    "VIRTUAL_ENV": str(old_root / ".venv"),
                    "PATH": f"{old_root / '.venv' / 'bin'}:/usr/bin",
                },
                "generation_id": 0,
                "peer_id": "gen0_peer0",
                "experiment_id": "relocated-task",
                "profile": "cpu",
                "cwd": str(old_root / "work"),
            }
            with patch.dict(
                os.environ,
                {
                    "PRAXIST_TASK_PROJECT_PATH": str(new_root),
                    "PRAXIST_WORKSPACE_ROOT": str(new_workspace),
                    "PRAXIST_DATASETS_DIR": str(new_workspace / "datasets"),
                    "PRAXIST_TASK_RUNTIME_ENV_KEYS": "TASK_DATA_ALIAS",
                    "TASK_DATA_ALIAS": str(new_workspace / "task_data"),
                    "PRAXIST_TASK_PYTHON": str(new_python),
                    "PRAXIST_TASK_VENV": str(new_root / ".venv"),
                    "VIRTUAL_ENV": str(new_root / ".venv"),
                    "PATH": f"{new_root / '.venv' / 'bin'}:/usr/bin",
                },
                clear=True,
            ):
                recovered = service._job_from_event("job", event, state="queued")

        self.assertEqual(recovered.command, [str(new_python), str(new_evaluator)])
        self.assertEqual(recovered.cwd, str(new_work))
        self.assertEqual(recovered.environment["PRAXIST_TASK_PROJECT_PATH"], str(new_root))
        self.assertEqual(recovered.environment["PRAXIST_WORKSPACE_ROOT"], str(new_workspace))
        self.assertEqual(
            recovered.environment["PRAXIST_DATASETS_DIR"], str(new_workspace / "datasets")
        )
        self.assertEqual(recovered.environment["TASK_DATA_ALIAS"], str(new_workspace / "task_data"))
        self.assertEqual(recovered.environment["EXTERNAL_DATA"], "/datasets/shared")
        self.assertEqual(recovered.environment["PRAXIST_TASK_PYTHON"], str(new_python))
        self.assertNotIn(str(old_root), json.dumps(recovered.environment, sort_keys=True))

    def test_recovered_task_relocation_does_not_rewrite_path_prefix_collisions(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            old_root = root / "task"
            new_root = root / "moved" / "task"
            prefix_collision = f"{old_root}_backup/results"

            command, environment, cwd = rebase_recovered_task_context(
                ["python", "evaluate.py", f"--external={prefix_collision}"],
                {
                    "PRAXIST_TASK_PROJECT_PATH": str(old_root),
                    "EXTERNAL_DATA": prefix_collision,
                },
                str(old_root / "work"),
                current_environment={"PRAXIST_TASK_PROJECT_PATH": str(new_root)},
            )

        self.assertEqual(command[-1], f"--external={prefix_collision}")
        self.assertEqual(environment["EXTERNAL_DATA"], prefix_collision)
        self.assertEqual(cwd, str(new_root / "work"))

    def test_recovered_task_rebases_runtime_locations_when_task_root_is_unchanged(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            task_root = root / "task"
            evaluator = task_root / "evaluations" / "run.py"
            old_python = root / "venvs" / "old" / "bin" / "python"
            new_python = root / "venvs" / "new" / "bin" / "python"
            old_data = root / "data-old"
            new_data = root / "data-new"
            evaluator.parent.mkdir(parents=True)
            evaluator.write_text("print('ok')\n", encoding="utf-8")
            service = ExperimentSchedulerService(
                run_dir=root / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            persisted_environment = {
                "PRAXIST_TASK_PROJECT_PATH": str(task_root),
                "PRAXIST_TASK_PYTHON": str(old_python),
                "PRAXIST_DATASETS_DIR": str(old_data),
            }
            current_environment = {
                "PRAXIST_TASK_PROJECT_PATH": str(task_root),
                "PRAXIST_TASK_PYTHON": str(new_python),
                "PRAXIST_DATASETS_DIR": str(new_data),
            }
            event = {
                "command": [str(old_python), str(evaluator)],
                "environment_values": persisted_environment,
                "generation_id": 0,
                "peer_id": "gen0_peer0",
                "experiment_id": "same-root-new-runtime",
                "profile": "cpu",
                "cwd": str(task_root),
            }

            self.assertTrue(
                task_runtime_context_changed(persisted_environment, current_environment)
            )
            with patch.dict(os.environ, current_environment, clear=True):
                recovered = service._job_from_event("job", event, state="queued")

        self.assertEqual(recovered.command[0], str(new_python))
        self.assertEqual(recovered.environment["PRAXIST_TASK_PYTHON"], str(new_python))
        self.assertEqual(recovered.environment["PRAXIST_DATASETS_DIR"], str(new_data))

    def test_recovered_missing_cwd_is_rejected_before_relaunch(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            task_root = root / "task"
            task_root.mkdir()
            service = ExperimentSchedulerService(
                run_dir=root / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            event = {
                "command": [sys.executable, "-c", "pass"],
                "environment_values": {"PRAXIST_TASK_PROJECT_PATH": str(task_root)},
                "generation_id": 0,
                "peer_id": "gen0_peer0",
                "experiment_id": "missing-recovery-cwd",
                "profile": "cpu",
                "cwd": str(root / "removed-work"),
            }
            recovered = service._job_from_event("job", event, state="queued")

            reason = service._recovery_rejection_reason(recovered)

        self.assertEqual(reason, "recovery_cwd_unavailable")

    def test_recovered_shell_task_runtime_normalization_is_idempotent(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            task_python = root / "task" / ".venv" / "bin" / "python"
            service = ExperimentSchedulerService(
                run_dir=root / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            submitted = service.submit(
                {
                    "command": ["bash", "-lc", "python evaluations/run.py"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "shell-recovery",
                    "environment": {
                        "PRAXIST_TASK_PYTHON": str(task_python),
                        "PATH": "/usr/bin",
                    },
                }
            )
            event = {
                **service._event_identity(submitted, include_request=True),
                "event": "submitted",
            }

            recovered = service._job_from_event(submitted.job_id, event, state="queued")

        self.assertEqual(recovered.command[2].count("PRAXIST_TASK_PYTHON="), 1)
        self.assertEqual(recovered.command[2].count("unset PYTHONPATH"), 1)
        self.assertEqual(recovered.command[2].count("unset PYTHONHOME"), 1)

    def test_task_python_binding_normalizes_only_supported_launch_forms(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            task_python = str(Path(td) / "task-python")
            environment = {"PRAXIST_TASK_PYTHON": task_python}
            cases = (
                (["python", "task.py"], [task_python, "task.py"]),
                (["python3", "task.py"], [task_python, "task.py"]),
                (["/usr/bin/env", "python", "task.py"], ["/usr/bin/env", task_python, "task.py"]),
                (
                    ["/usr/bin/env", "--", "python", "task.py"],
                    ["/usr/bin/env", "--", task_python, "task.py"],
                ),
                (
                    ["/usr/bin/env", "-u", "HOME", "python3", "task.py"],
                    ["/usr/bin/env", "-u", "HOME", task_python, "task.py"],
                ),
                (
                    ["/usr/bin/env", "-i", "MODE=test", "python3", "task.py"],
                    ["/usr/bin/env", "-i", "MODE=test", task_python, "task.py"],
                ),
                (["/usr/bin/env", "--"], ["/usr/bin/env", "--"]),
                ([sys.executable, "task.py"], [sys.executable, "task.py"]),
                (["python3.11", "task.py"], [task_python, "task.py"]),
                (
                    ["/usr/bin/env", "python3.11", "task.py"],
                    ["/usr/bin/env", task_python, "task.py"],
                ),
                (["/usr/bin/python3", "task.py"], ["/usr/bin/python3", "task.py"]),
            )
            for index, (command, expected) in enumerate(cases):
                with self.subTest(command=command):
                    job = service.submit(
                        {
                            "command": command,
                            "peer_id": "gen0_peer0",
                            "generation_id": 0,
                            "experiment_id": f"python-binding-{index}",
                            "environment": environment,
                        }
                    )
                    self.assertEqual(job.command, expected)

            shell = service.submit(
                {
                    "command": ["/bin/bash", "-lc", "python3 -c 'pass'"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "python-binding-shell",
                    "environment": environment,
                }
            )
            self.assertEqual(shell.command[:2], ["/bin/bash", "-lc"])
            self.assertIn("unset PYTHONPATH", shell.command[2])
            self.assertIn("unset PYTHONHOME", shell.command[2])
            self.assertIn(f"PRAXIST_TASK_PYTHON={task_python}", shell.command[2])
            self.assertIn('python3() { command "$PRAXIST_TASK_PYTHON" "$@"; }', shell.command[2])
            self.assertTrue(shell.command[2].endswith(f"{task_python} -c 'pass'"))

            script_invocation = service.submit(
                {
                    "command": ["/bin/bash", "task.sh", "-c", "python3 task.py"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "python-binding-shell-script",
                    "environment": environment,
                }
            )
            self.assertEqual(
                script_invocation.command,
                ["/bin/bash", "task.sh", "-c", "python3 task.py"],
            )

            shell_with_option = service.submit(
                {
                    "command": ["/bin/bash", "-O", "extglob", "-lc", "python task.py"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "python-binding-shell-option",
                    "environment": environment,
                }
            )
            self.assertIn("PRAXIST_TASK_PYTHON", shell_with_option.command[-1])

            incomplete_shell_option = service.submit(
                {
                    "command": ["/bin/bash", "-O"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "python-binding-incomplete-shell-option",
                    "environment": environment,
                }
            )
            self.assertEqual(incomplete_shell_option.command, ["/bin/bash", "-O"])

    def test_task_child_drops_runner_python_paths_unless_task_declares_them(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            task_root = Path(td).resolve() / "task"
            task_root.mkdir()
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            base_environment = {
                "PRAXIST_TASK_PROJECT_PATH": str(task_root),
                "PRAXIST_TASK_PYTHON": str(task_root / ".venv" / "bin" / "python"),
                "PYTHONPATH": "/runner/site-packages",
                "PYTHONHOME": "/runner/python",
            }
            isolated = service.submit(
                {
                    "command": ["python", "evaluations/run.py"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "isolated-python-env",
                    "environment": base_environment,
                }
            )
            self.assertNotIn("PYTHONPATH", isolated.environment)
            self.assertNotIn("PYTHONHOME", isolated.environment)
            self.assertEqual(
                isolated.environment["PATH"].split(os.pathsep)[0],
                str(task_root / ".venv" / "bin"),
            )
            self.assertEqual(isolated.cwd, str(task_root.resolve()))

            task_owned = service.submit(
                {
                    "command": ["python", "evaluations/run.py"],
                    "peer_id": "gen0_peer1",
                    "generation_id": 0,
                    "experiment_id": "task-owned-python-env",
                    "environment": {
                        **base_environment,
                        "PRAXIST_TASK_RUNTIME_ENV_KEYS": "PYTHONHOME,PYTHONPATH",
                    },
                }
            )
            self.assertEqual(task_owned.environment["PYTHONPATH"], "/runner/site-packages")
            self.assertEqual(task_owned.environment["PYTHONHOME"], "/runner/python")

    def test_launch_barrier_ignores_task_pythonpath_but_task_process_receives_it(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            run_dir = root / "run"
            task_root = root / "task"
            shadow_dir = task_root / "pythonpath"
            output = task_root / "observed-pythonpath.txt"
            shadow_dir.mkdir(parents=True)
            (shadow_dir / "json.py").write_text(
                "raise RuntimeError('task PYTHONPATH reached scheduler barrier')\n",
                encoding="utf-8",
            )
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            service.start()
            try:
                with patch.dict(
                    os.environ,
                    {
                        "PRAXIST_TASK_PROJECT_PATH": str(task_root),
                        "PRAXIST_TASK_PYTHON": sys.executable,
                        "PRAXIST_TASK_RUNTIME_ENV_KEYS": "PYTHONPATH",
                        "PYTHONPATH": str(shadow_dir),
                        "TASK_OUTPUT": str(output),
                    },
                ):
                    code = submit_and_wait(
                        [
                            sys.executable,
                            "-c",
                            (
                                "import os,pathlib;"
                                "pathlib.Path(os.environ['TASK_OUTPUT']).write_text("
                                "os.environ['PYTHONPATH'])"
                            ),
                        ],
                        peer_id="gen0_peer0",
                        experiment_id="task-owned-pythonpath",
                        run_dir=run_dir,
                    )
            finally:
                service.stop()

            self.assertEqual(code, 0)
            self.assertEqual(output.read_text(encoding="utf-8"), str(shadow_dir))

    def test_scheduler_rejects_missing_explicit_experiment_cwd_at_admission(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            with self.assertRaisesRegex(ExperimentRejected, "cwd is not an existing directory"):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "missing-cwd",
                        "cwd": str(Path(td) / "missing"),
                    }
                )

    def test_failure_classification_keeps_resource_and_permission_distinct(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend.experiment_scheduler import (
            _classify_failure,
        )

        self.assertEqual(_classify_failure(1, "resource exhausted"), "resource")
        self.assertEqual(_classify_failure(1, "permission denied"), "permission")

    def test_task_python_binding_reaches_bash_login_shell_process_boundary(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            run_dir = root / "run"
            marker = root / "task-python-used"
            wrapper_dir = root / "task environment"
            wrapper_dir.mkdir()
            task_python = wrapper_dir / "python"
            task_python.write_text(
                f"#!{sys.executable}\n"
                "import os, pathlib, sys\n"
                "pathlib.Path(os.environ['WRAPPER_MARKER']).write_text('used')\n"
                "os.execv(sys.executable, [sys.executable, *sys.argv[1:]])\n",
                encoding="utf-8",
            )
            task_python.chmod(0o755)
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = service.submit(
                {
                    "command": ["/bin/bash", "-lc", "python3 -c 'pass'"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "bash-login-task-python",
                    "environment": {
                        **os.environ,
                        "PRAXIST_TASK_PYTHON": str(task_python),
                        "WRAPPER_MARKER": str(marker),
                    },
                }
            )
            service.start()
            try:
                result = service.wait(job.job_id, 5)["job"]
            finally:
                service.stop()
            self.assertEqual(result["state"], "completed", result)
            self.assertEqual(marker.read_text(encoding="utf-8"), "used")

    def test_failed_job_status_retains_redacted_bounded_explanation(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            secret_sample = Path(td) / "failure-sample.txt"
            secret_sample.write_text("API_KEY=supersecretvalue", encoding="utf-8")
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = service.submit(
                {
                    "command": [
                        sys.executable,
                        "-c",
                        (
                            "import pathlib,sys;print('x'*6000);"
                            "print(pathlib.Path(sys.argv[1]).read_text());"
                            "print('ModuleNotFoundError: missing_widget');sys.exit(2)"
                        ),
                        str(secret_sample),
                    ],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "explained-failure",
                }
            )
            service.start()
            try:
                result = service.wait(job.job_id, 5)["job"]
                status = service.status()
            finally:
                service.stop()
            self.assertEqual(result["state"], "failed")
            self.assertEqual(result["failure_category"], "environment")
            self.assertEqual(result["failure_signature"], "ModuleNotFoundError: missing_widget")
            self.assertEqual(result["error"], result["failure_signature"])
            self.assertLessEqual(len(result["failure_log_tail"]), 4096)
            self.assertIn("<redacted:named_secret>", result["failure_log_tail"])
            self.assertNotIn("supersecretvalue", result["failure_log_tail"])
            self.assertEqual(status["failure_counts_by_category"], {"environment": 1})
            self.assertEqual(status["jobs_total"], 1)
            self.assertEqual(status["jobs_retained"], 1)
            self.assertEqual(status["jobs_omitted"], 0)

            events_path = run_dir / "resource_scheduler" / "events.jsonl"
            events_text = events_path.read_text(encoding="utf-8")
            self.assertNotIn("supersecretvalue", events_text)
            terminal = [
                json.loads(line)
                for line in events_text.splitlines()
                if json.loads(line).get("event") == "completed"
            ][-1]
            self.assertEqual(terminal["failure_category"], "environment")
            self.assertEqual(terminal["failure_signature"], result["failure_signature"])

            recovered = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            recovered._recover_terminal_events()
            recovered_job = recovered.status()["jobs"][0]
            self.assertEqual(recovered_job["failure_category"], "environment")
            self.assertEqual(recovered_job["failure_signature"], result["failure_signature"])

    def test_status_reports_total_and_retained_job_counts_separately(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            for index in range(3):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": f"gen0_peer{index}",
                        "generation_id": 0,
                        "experiment_id": f"retention-{index}",
                    }
                )
            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler._JOB_DETAIL_LIMIT",
                2,
            ):
                status = service.status()
            self.assertEqual(status["jobs_total"], 3)
            self.assertEqual(status["jobs_retained"], 2)
            self.assertEqual(status["jobs_omitted"], 1)
            self.assertEqual(len(status["jobs"]), 2)

    def test_unbound_command_secret_is_rejected_before_persistence(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            for index, option in enumerate(
                (
                    "--token",
                    "--access-token",
                    "--client-secret",
                    "--private-key",
                    "--connection-string",
                )
            ):
                service = ExperimentSchedulerService(
                    run_dir=Path(td) / f"run-{index}",
                    settings=_settings(maximum=1),
                    allocator=_GPUAllocator(""),
                )
                with (
                    self.subTest(option=option),
                    self.assertRaisesRegex(ExperimentRejected, "credential-bearing"),
                ):
                    service.submit(
                        {
                            "command": [sys.executable, "task.py", option, "opaque-value"],
                            "peer_id": "gen0_peer0",
                            "generation_id": 0,
                            "experiment_id": f"unsafe-command-secret-{index}",
                            "profile": "cpu",
                            "environment": {},
                        }
                    )
                self.assertFalse((service.state_dir / "events.jsonl").exists())

    def test_second_scheduler_cannot_take_over_live_run_owner(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("", limit=1),
            )
            second = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            first.start()
            try:
                endpoint = first.endpoint
                self.assertTrue(endpoint.exists())
                with self.assertRaisesRegex(RuntimeError, "already owns run"):
                    second.start()
                self.assertTrue(endpoint.exists())
                self.assertEqual(first.handle_request({"action": "ping"})["run_id"], run_dir.name)
            finally:
                first.stop()

    def test_durable_submitted_event_recovers_acknowledged_queue(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            marker = Path(td) / "executed"
            interrupted = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            submitted = interrupted.submit(
                {
                    "command": [
                        sys.executable,
                        "-c",
                        "import pathlib,sys;pathlib.Path(sys.argv[1]).write_text('once')",
                        str(marker),
                    ],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "durable-queue-entry",
                    "profile": "cpu",
                }
            )
            events = (interrupted.state_dir / "events.jsonl").read_text(encoding="utf-8")
            self.assertIn('"event": "submitted"', events)

            resumed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            resumed.start()
            try:
                duplicate = resumed.submit(
                    {
                        "command": [sys.executable, "-c", "raise SystemExit(99)"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "durable-queue-entry",
                        "profile": "cpu",
                    }
                )
                self.assertEqual(duplicate.job_id, submitted.job_id)
                result = resumed.wait(submitted.job_id, 5)["job"]
            finally:
                resumed.stop()
            self.assertEqual(result["state"], "completed")
            self.assertEqual(marker.read_text(), "once")

    def test_cancelled_queue_entry_is_not_resurrected_after_restart(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            marker = Path(td) / "must-not-run"
            interrupted = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            submitted = interrupted.submit(
                {
                    "command": [
                        sys.executable,
                        "-c",
                        "import pathlib,sys;pathlib.Path(sys.argv[1]).write_text('bad')",
                        str(marker),
                    ],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "cancel-before-launch",
                    "profile": "cpu",
                }
            )
            self.assertTrue(interrupted.cancel_queued(submitted.job_id)["cancelled"])

            resumed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            resumed.start()
            try:
                self.assertEqual(resumed.status()["queued"], 0)
                self.assertFalse(marker.exists())
                replacement = resumed.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "cancel-before-launch",
                        "profile": "cpu",
                    }
                )
                self.assertNotEqual(replacement.job_id, submitted.job_id)
                self.assertEqual(resumed.wait(replacement.job_id, 5)["job"]["state"], "completed")
            finally:
                resumed.stop()

    def test_assessment_and_freeze_fences_survive_restart(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            interrupted = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            interrupted.open_generation(3, deadline=time.time() + 600, cohort_size=2)
            interrupted.begin_assessment(3, "evidence-ready")
            interrupted.freeze_generation(4, "closed")

            resumed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            resumed.start()
            try:
                self.assertIn(3, resumed.status()["assessment_generations"])
                self.assertIn(4, resumed.status()["frozen_generations"])
                with self.assertRaisesRegex(ExperimentRejected, "only mature work"):
                    resumed.submit(
                        {
                            "command": [sys.executable, "-c", "pass"],
                            "peer_id": "gen3_peer0",
                            "generation_id": 3,
                            "experiment_id": "ordinary-after-restart",
                            "profile": "cpu",
                        }
                    )
                with self.assertRaisesRegex(ExperimentRejected, "is closing"):
                    resumed.submit(
                        {
                            "command": [sys.executable, "-c", "pass"],
                            "peer_id": "gen4_peer0",
                            "generation_id": 4,
                            "experiment_id": "frozen-after-restart",
                            "profile": "cpu",
                        }
                    )
            finally:
                resumed.stop()

    def test_launch_intent_finds_delayed_ready_process_without_requeue(self) -> None:
        if not Path("/proc").is_dir():
            self.skipTest("/proc filesystem required for process cmdline inspection")
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            interrupted = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = interrupted.submit(
                {
                    "command": [sys.executable, "-c", "raise SystemExit(99)"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "delayed-ready",
                    "profile": "cpu",
                }
            )
            job.attempts = 1
            attempt_dir = interrupted.state_dir / "attempts" / f"{job.job_id}-a1"
            attempt_dir.mkdir(parents=True)
            interrupted._append_event(
                {
                    "event": "launch_intent",
                    **interrupted._event_identity(job, include_request=True),
                    "allocation_id": "delayed-allocation",
                    "gpu_uuids": [],
                    "attempt_id": f"{job.job_id}-a1",
                    "attempt_dir": str(attempt_dir),
                },
                required=True,
            )
            process = subprocess.Popen(
                [
                    sys.executable,
                    "-c",
                    "import time;time.sleep(30)",
                    str(attempt_dir / "READY.json"),
                    f"{job.job_id}-a1",
                ],
                start_new_session=True,
            )
            resumed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            try:
                resumed.start()
                self.assertIn(job.job_id, resumed._active)
                self.assertNotIn(job.job_id, resumed._queue)
                self.assertTrue((attempt_dir / "GO.json").exists())
                os.killpg(process.pid, signal.SIGTERM)
                process.wait(timeout=5)
                self.assertEqual(resumed.wait(job.job_id, 5)["job"]["state"], "drained_unknown")
            finally:
                if process.poll() is None:
                    os.killpg(process.pid, signal.SIGKILL)
                    process.wait(timeout=5)
                resumed.stop()

    def test_new_attempt_does_not_inherit_previous_attempt_pid(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "attempt-reset",
                    "profile": "cpu",
                }
            )
            attempt_dir = service.state_dir / "attempts" / f"{job.job_id}-a2"
            attempt_dir.mkdir(parents=True)
            service._append_event(
                {
                    "event": "process_started",
                    **service._event_identity(job, include_request=True),
                    "attempt": 1,
                    "pid": 111,
                    "pgid": 111,
                    "pid_start_time": 1,
                },
                required=True,
            )
            job.attempts = 2
            service._append_event(
                {
                    "event": "launch_intent",
                    **service._event_identity(job, include_request=True),
                    "allocation_id": "attempt-2-allocation",
                    "attempt_id": f"{job.job_id}-a2",
                    "attempt_dir": str(attempt_dir),
                    "gpu_uuids": [],
                },
                required=True,
            )
            with (
                patch.object(service, "_read_ready_process", return_value={}) as read_ready,
                patch.object(service, "_find_attempt_process", return_value={}) as find_process,
            ):
                service._recover_terminal_events()
            read_ready.assert_called_once()
            find_process.assert_called_once()
            self.assertIn(job.job_id, service._queue)

    def test_replay_restores_submitted_retry_terminal_and_generation_controls(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
            )
            common = {
                "generation_id": 2,
                "profile": "cpu",
                "work_class": "ordinary",
                "command": [sys.executable, "-c", "pass"],
                "environment_values": {},
                "attempt": 0,
                "submitted_at": time.time(),
            }
            events = [
                {
                    "event": "generation_open",
                    "generation_id": 2,
                    "deadline": time.time() + 600,
                    "cohort_size": 2,
                },
                {
                    "event": "submitted",
                    **common,
                    "job_id": "submitted-job",
                    "peer_id": "gen2_peer0",
                    "experiment_id": "submitted-science",
                },
                {
                    "event": "retry_queued",
                    **common,
                    "job_id": "retry-job",
                    "peer_id": "gen2_peer1",
                    "experiment_id": "retry-science",
                    "attempt": 1,
                },
                {
                    "event": "completed",
                    **common,
                    "job_id": "completed-job",
                    "peer_id": "gen2_peer2",
                    "experiment_id": "completed-science",
                    "state": "completed",
                    "exit_code": 0,
                },
            ]
            (service.state_dir / "events.jsonl").write_text(
                "\n".join(json.dumps(event) for event in events) + "\n",
                encoding="utf-8",
            )
            service._recover_terminal_events()
            self.assertEqual(set(service._queue), {"submitted-job", "retry-job"})
            self.assertEqual(service._jobs["completed-job"].state, "completed")
            self.assertEqual(service._generation_cohort_sizes[2], 2)

            service.begin_assessment(2)
            service.freeze_generation(2)
            service.freeze_all()
            replayed = ExperimentSchedulerService(
                run_dir=service.run_dir,
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
            )
            replayed._recover_terminal_events()
            self.assertIn(2, replayed._assessment_generations)
            self.assertIn(2, replayed._frozen_generations)
            self.assertTrue(replayed._admission_closed)

    def test_resume_clears_rerun_signal_before_scheduler_recovers_submitted_job(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend.generation_resume import (
            prepare_resume_for_sidecars,
        )

        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
            )
            submitted = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "resume-after-close",
                    "profile": "cpu",
                    "eta_seconds": 10,
                }
            )
            gen_dir = run_dir / "gen_0"
            gen_dir.mkdir()
            (gen_dir / "CLOSING_SIGNAL").write_text("stale\n", encoding="utf-8")
            service._append_event(
                {"event": "generation_frozen", "generation_id": 0, "reason": "closing"},
                required=True,
            )
            service._append_event(
                {"event": "admission_closed", "reason": "scheduler_stopped"},
                required=True,
            )

            plan = prepare_resume_for_sidecars(
                run_dir,
                max_generations=2,
                pi_enabled=True,
                policy="completed_generation",
            )
            replayed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
                recovery_rerun_generation=plan.start_generation,
            )
            replayed._recover_terminal_events()

            self.assertFalse(plan.has_pending_boundary)
            self.assertFalse((gen_dir / "CLOSING_SIGNAL").exists())
            self.assertEqual(replayed._jobs[submitted.job_id].state, "queued")
            self.assertIn(submitted.job_id, replayed._queue)
            self.assertTrue(replayed._admission_closed)
            self.assertIn(0, replayed._frozen_generations)
            replayed.open_generation(0, deadline=time.time() + 60, cohort_size=1)
            self.assertFalse(replayed._admission_closed)
            self.assertNotIn(0, replayed._frozen_generations)

    def test_resume_rerun_releases_semantic_identity_rejected_by_prior_close(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
            )
            first = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "closed-before-launch",
                    "profile": "cpu",
                }
            )
            service.freeze_generation(0, "interrupted")
            self.assertEqual(first.state, "rejected")

            replayed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
                recovery_rerun_generation=0,
            )
            replayed._recover_terminal_events()
            replayed.open_generation(0, deadline=time.time() + 60, cohort_size=1)
            replacement = replayed.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "closed-before-launch",
                    "profile": "cpu",
                }
            )

            self.assertNotEqual(replacement.job_id, first.job_id)
            self.assertEqual(replacement.state, "queued")

    def test_resume_rerun_accepts_prior_close_control_error_spellings(self) -> None:
        control_errors = (
            "admission_closed",
            "admission_closed:operator_stop",
            "generation_closing",
            "generation_closing:boundary",
            "generation_assessment",
            "generation_assessment:quorum_reached",
        )
        for error in control_errors:
            with self.subTest(error=error), tempfile.TemporaryDirectory() as td:
                run_dir = Path(td) / "run"
                service = ExperimentSchedulerService(
                    run_dir=run_dir,
                    settings=_settings(maximum=2),
                    allocator=_GPUAllocator("", limit=2),
                )
                first = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "closed-control-spelling",
                        "profile": "cpu",
                    }
                )
                service._finish_without_launch(first, error)

                replayed = ExperimentSchedulerService(
                    run_dir=run_dir,
                    settings=_settings(maximum=2),
                    allocator=_GPUAllocator("", limit=2),
                    recovery_rerun_generation=0,
                )
                replayed._recover_terminal_events()
                replayed.open_generation(0, deadline=time.time() + 60, cohort_size=1)
                replacement = replayed.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "closed-control-spelling",
                        "profile": "cpu",
                    }
                )

                self.assertNotEqual(replacement.job_id, first.job_id)
                self.assertEqual(replacement.state, "queued")

    def test_resume_rerun_preserves_unrelated_terminal_semantic_identity(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
            )
            first = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "unrelated-rejection",
                    "profile": "cpu",
                }
            )
            service._finish_without_launch(first, "task_policy_rejected")

            replayed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
                recovery_rerun_generation=0,
            )
            replayed._recover_terminal_events()
            replayed.open_generation(0, deadline=time.time() + 60, cohort_size=1)
            duplicate = replayed.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "unrelated-rejection",
                    "profile": "cpu",
                }
            )

            self.assertEqual(duplicate.job_id, first.job_id)
            self.assertEqual(duplicate.state, "rejected")

    def test_resume_rerun_never_releases_completed_or_failed_identity_by_error_text(self) -> None:
        for state, exit_code in (("completed", 0), ("failed", 1)):
            with self.subTest(state=state), tempfile.TemporaryDirectory() as td:
                run_dir = Path(td) / "run"
                service = ExperimentSchedulerService(
                    run_dir=run_dir,
                    settings=_settings(maximum=2),
                    allocator=_GPUAllocator("", limit=2),
                )
                first = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "terminal-with-close-text",
                        "profile": "cpu",
                    }
                )
                service._append_event(
                    {
                        "event": "completed",
                        **service._event_identity(first),
                        "state": state,
                        "exit_code": exit_code,
                        "error": "generation_closing",
                    },
                    required=True,
                )

                replayed = ExperimentSchedulerService(
                    run_dir=run_dir,
                    settings=_settings(maximum=2),
                    allocator=_GPUAllocator("", limit=2),
                    recovery_rerun_generation=0,
                )
                replayed._recover_terminal_events()
                replayed.open_generation(0, deadline=time.time() + 60, cohort_size=1)
                duplicate = replayed.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "terminal-with-close-text",
                        "profile": "cpu",
                    }
                )

                self.assertEqual(duplicate.job_id, first.job_id)
                self.assertEqual(duplicate.state, state)

    def test_recovery_and_timing_helpers_cover_conservative_boundaries(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("GPU-a", limit=2),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen1_peer0",
                    "generation_id": 1,
                    "experiment_id": "helper-boundaries",
                    "profile": "cpu",
                    "eta_seconds": 10,
                }
            )
            self.assertEqual(service._recovery_rejection_reason(job), "")
            service._admission_closed = True
            self.assertEqual(service._recovery_rejection_reason(job), "recovery_admission_closed")
            service._admission_closed = False
            service._frozen_generations.add(1)
            self.assertEqual(service._recovery_rejection_reason(job), "recovery_generation_frozen")
            service._frozen_generations.clear()
            service._assessment_generations.add(1)
            self.assertEqual(
                service._recovery_rejection_reason(job), "recovery_generation_assessment"
            )
            job.work_class = "mature"
            self.assertEqual(service._recovery_rejection_reason(job), "")

            with patch.object(Path, "read_bytes", side_effect=OSError("gone")):
                self.assertEqual(service._read_process_environment(1), {})
            with patch.object(Path, "read_bytes", return_value=b"X" * (1024 * 1024 + 1)):
                self.assertEqual(service._read_process_environment(1), {})
            with patch.object(
                Path,
                "read_bytes",
                return_value=b"GOOD=value\0missing-separator\0BAD=\xff\0",
            ):
                self.assertEqual(service._read_process_environment(1), {"GOOD": "value"})

            service.allocator.snapshot = HostSnapshot(
                8,
                5,
                5,
                0,
                gpus=(GPUDevice(0, "GPU-a", 40960, 0, 0),),
                gpu_processes=(GPUProcess(22, "GPU-a", 1024, 25),),
            )
            self.assertEqual(
                service._recovered_gpu_uuids(
                    {"PRAXIST_ASSIGNED_GPU_UUIDS": "GPU-a"}, pid=1, pgid=1
                ),
                ("GPU-a",),
            )
            self.assertEqual(
                service._recovered_gpu_uuids({"CUDA_VISIBLE_DEVICES": "0"}, pid=1, pgid=1),
                ("GPU-a",),
            )
            with patch.object(os, "getpgid", return_value=7):
                self.assertEqual(
                    service._recovered_gpu_uuids({}, pid=1, pgid=7),
                    ("GPU-a",),
                )
            with patch.object(os, "getpgid", side_effect=OSError("gone")):
                self.assertEqual(
                    service._recovered_gpu_uuids({}, pid=22, pgid=7),
                    ("GPU-a",),
                )

            self.assertEqual(
                service._recovered_profile({"PRAXIST_RESOURCE_PROFILE": "gpu"}, ("GPU-a",)).name,
                "gpu",
            )
            self.assertEqual(service._recovered_profile({}, ()).name, "cpu")
            service.settings.profiles = {"gpu": service.settings.profiles["gpu"]}
            service.settings.default_profile = "gpu"
            self.assertEqual(service._recovered_profile({}, ()).name, "unclassified_recovery")

            self.assertFalse(
                service._event_process_matches({"pid": 0, "pgid": 0}, attempt_dir=None)
            )
            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler._pid_start_time",
                return_value=5,
            ):
                self.assertTrue(
                    service._event_process_matches(
                        {"pid": 1, "pgid": 1, "pid_start_time": 5}, attempt_dir=None
                    )
                )
                self.assertFalse(
                    service._event_process_matches(
                        {"pid": 1, "pgid": 1, "pid_start_time": "bad"}, attempt_dir=None
                    )
                )
            with (
                patch.object(Path, "is_dir", return_value=False),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler._pid_start_time",
                    return_value="ps:stable-start",
                ),
            ):
                self.assertTrue(
                    service._event_process_matches(
                        {"pid": 1, "pgid": 1, "pid_start_time": "ps:stable-start"},
                        attempt_dir=Path(td),
                    )
                )
                self.assertFalse(
                    service._event_process_matches({"pid": 1, "pgid": 1}, attempt_dir=Path(td))
                )
            self.assertFalse(
                service._event_process_matches({"pid": 1, "pgid": 1}, attempt_dir=None)
            )
            with patch.object(Path, "read_bytes", side_effect=OSError("gone")):
                self.assertFalse(
                    service._event_process_matches({"pid": 1, "pgid": 1}, attempt_dir=Path(td))
                )

            with self.assertRaisesRegex(RuntimeError, "unavailable"):
                service._restore_durable_command(["__PRAXIST_ENV_REF_TOKEN__"], {})
            service._queue.remove(job.job_id)
            job.supply_claim_id = "claim"
            service._finish_without_launch(job, "done")
            self.assertEqual(job.state, "rejected")

            job.state = "running"
            job.work_class = "ordinary"
            service.settings.deadline_admission = True
            service._generation_deadlines[1] = time.time() - 1
            self.assertFalse(service._fits_generation(job))
            job.eta_seconds = 0
            service._observe_runtime(job, started_at=time.time())
            self.assertEqual(service._completion_probability(job, deadline=time.time() + 10), 0)
            job.eta_seconds = 10
            service._observe_runtime(job, started_at=time.time() - 5)
            service._observe_runtime(job, started_at=time.time() - 20)
            self.assertGreater(service._completion_probability(job, deadline=time.time() + 10), 0)

    def test_scheduler_defensive_lifecycle_and_supply_branches(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            allocator = _GPUAllocator("", limit=2)
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=2),
                allocator=allocator,
            )
            service._acquire_owner_lock()
            with self.assertRaisesRegex(RuntimeError, "already started"):
                service._acquire_owner_lock()
            service._release_owner_lock()
            service._release_owner_lock()

            queued = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "stop-queued",
                }
            )
            service._acquire_owner_lock()
            service.stop()
            self.assertEqual(queued.state, "rejected")

            failing_allocator = MagicMock()
            failing = ExperimentSchedulerService(
                run_dir=Path(td) / "failing",
                settings=_settings(maximum=1),
                allocator=failing_allocator,
            )
            with (
                patch.object(failing, "_recover_terminal_events", side_effect=RuntimeError("boom")),
                self.assertRaisesRegex(RuntimeError, "boom"),
            ):
                failing.start()
            failing_allocator.close.assert_called_once()
            self.assertIsNone(failing._owner_lock_fd)

            service = ExperimentSchedulerService(
                run_dir=Path(td) / "supply",
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
            )
            service.settings.supply_idle_samples = 2
            service.open_generation(1, deadline=time.time() + 600, cohort_size=2)
            service.configure_generation_maturity(
                1,
                cohort_size=2,
                mature_target=1,
                count_callback=lambda: (_ for _ in ()).throw(RuntimeError("unknown")),
            )
            service._refresh_mature_counts(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            self.assertIsNone(service._mature_completed[1])
            self.assertEqual(service.generation_advice("bad", 1), {})
            self.assertEqual(service.generation_advice("gen1_peer0", 99), {})
            service._frozen_generations.add(1)
            service.begin_assessment(1)
            service._frozen_generations.clear()

            service.register_idle_supply("gen1_peer0", 1)
            first = HostSnapshot(8, 5, 5, 0, observed_at=2.0)
            service._reconcile_supply(first)
            self.assertEqual(service._supply_leases, {})
            service._reconcile_supply(first)
            second = HostSnapshot(8, 5, 5, 0, observed_at=3.0)
            service._reconcile_supply(second)
            self.assertEqual(len(service._supply_leases), 1)
            lease_id = next(iter(service._supply_leases))
            self.assertEqual(service.register_idle_supply("gen1_peer0", 1)["lease_id"], lease_id)
            self.assertEqual(service.get_supply_lease("bad", 1, lease_id), {})
            self.assertEqual(service.get_supply_lease("gen1_peer0", 1, "missing"), {})
            service.release_supply_lease(lease_id, "wrong-peer")
            self.assertIn(lease_id, service._supply_leases)
            service.unregister_idle_supply("gen1_peer0", 1)
            self.assertNotIn(lease_id, service._supply_leases)

            service.register_idle_supply("gen1_peer1", 1)
            service.allocator.has_supply_headroom = lambda *_args, **_kwargs: False
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=4.0))
            self.assertEqual(service._supply_idle_samples, 0)
            service.freeze_generation(1)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=5.0))
            self.assertNotIn("gen1_peer1", service._idle_supply_waiters)

    def test_replay_handles_invalid_credentials_empty_requests_and_closed_generations(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("", limit=2),
            )
            base = {
                "generation_id": 0,
                "profile": "cpu",
                "work_class": "ordinary",
                "environment_values": {},
                "attempt": 0,
                "submitted_at": time.time(),
            }
            events = [
                {
                    "event": "submitted",
                    **base,
                    "job_id": "valid",
                    "peer_id": "gen0_peer0",
                    "experiment_id": "valid",
                    "command": [sys.executable, "-c", "pass"],
                },
                {
                    "event": "submitted",
                    **base,
                    "job_id": "empty",
                    "peer_id": "gen0_peer1",
                    "experiment_id": "empty",
                    "command": [],
                },
                {
                    "event": "retry_queued",
                    **base,
                    "job_id": "changed-secret",
                    "peer_id": "gen0_peer2",
                    "experiment_id": "changed-secret",
                    "command": [sys.executable, "-c", "pass"],
                    "environment_sensitive_hashes": {"SERVICE_TOKEN": "not-current"},
                },
                {
                    "event": "retry_queued",
                    **base,
                    "job_id": "valid-retry",
                    "peer_id": "gen0_peer3",
                    "experiment_id": "valid-retry",
                    "command": [sys.executable, "-c", "pass"],
                    "attempt": 1,
                },
            ]
            (service.state_dir / "events.jsonl").write_text(
                "malformed\n" + "\n".join(json.dumps(event) for event in events) + "\n",
                encoding="utf-8",
            )
            with patch.dict(os.environ, {"SERVICE_TOKEN": "current"}, clear=True):
                service._recover_terminal_events()
            self.assertEqual(set(service._queue), {"valid", "valid-retry"})
            self.assertNotIn("empty", service._jobs)
            self.assertEqual(service._jobs["changed-secret"].state, "failed")

            frozen = ExperimentSchedulerService(
                run_dir=Path(td) / "frozen",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("", limit=1),
            )
            frozen._frozen_generations.add(4)
            frozen_event = {
                "event": "submitted",
                "job_id": "frozen-job",
                "generation_id": 4,
                "peer_id": "gen4_peer0",
                "experiment_id": "frozen-job",
                "profile": "cpu",
                "work_class": "ordinary",
                "command": [sys.executable, "-c", "pass"],
                "environment_values": {},
            }
            (frozen.state_dir / "events.jsonl").write_text(
                json.dumps(frozen_event) + "\n", encoding="utf-8"
            )
            frozen._recover_terminal_events()
            self.assertEqual(frozen._jobs["frozen-job"].state, "rejected")

    def test_attempt_discovery_and_identity_validation_paths(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            attempt_dir = Path(td) / "attempt"
            attempt_dir.mkdir()
            ready_path = attempt_dir / "READY.json"
            payload = json.dumps(
                {"pid": 123, "pgid": 123, "attempt_id": "attempt-a1", "pid_start_time": 5}
            )
            command = b"python\0" + str(ready_path).encode() + b"\0attempt-a1\0"
            with (
                patch.object(Path, "read_text", return_value=payload),
                patch.object(Path, "read_bytes", return_value=command),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler._pid_start_time",
                    return_value=5,
                ),
            ):
                self.assertEqual(
                    ExperimentSchedulerService._read_ready_process(attempt_dir, "attempt-a1"),
                    {"pid": 123, "pgid": 123},
                )

            portable_payload = json.dumps(
                {
                    "pid": 123,
                    "pgid": 123,
                    "attempt_id": "attempt-a1",
                    "pid_start_time": "ps:stable-start",
                }
            )
            with (
                patch.object(Path, "read_text", return_value=portable_payload),
                patch.object(Path, "is_dir", return_value=False),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler._pid_start_time",
                    return_value="ps:stable-start",
                ),
            ):
                self.assertEqual(
                    ExperimentSchedulerService._read_ready_process(attempt_dir, "attempt-a1"),
                    {"pid": 123, "pgid": 123},
                )

            for invalid_portable in (
                {**json.loads(portable_payload), "attempt_id": "other"},
                {**json.loads(portable_payload), "pid_start_time": None},
            ):
                with (
                    self.subTest(portable_payload=invalid_portable),
                    patch.object(Path, "read_text", return_value=json.dumps(invalid_portable)),
                    patch.object(Path, "is_dir", return_value=False),
                    patch(
                        "praxist.plugins.workflow_stages.research_loop.backend."
                        "experiment_scheduler._pid_start_time",
                        return_value="ps:stable-start",
                    ),
                    patch(
                        "praxist.plugins.workflow_stages.research_loop.backend."
                        "experiment_scheduler.time.monotonic",
                        side_effect=[0.0, 0.0, 3.0],
                    ),
                    patch(
                        "praxist.plugins.workflow_stages.research_loop.backend."
                        "experiment_scheduler.time.sleep"
                    ),
                ):
                    self.assertEqual(
                        ExperimentSchedulerService._read_ready_process(attempt_dir, "attempt-a1"),
                        {},
                    )

            invalid_payloads = [
                ({**json.loads(payload), "attempt_id": "other"}, command),
                (json.loads(payload), b"python\0" + str(ready_path).encode() + b"\0"),
                ({**json.loads(payload), "pid_start_time": 6}, command),
                (json.loads(payload), b"python\0wrong\0"),
            ]
            for invalid_payload, invalid_command in invalid_payloads:
                with (
                    self.subTest(payload=invalid_payload),
                    patch.object(Path, "read_text", return_value=json.dumps(invalid_payload)),
                    patch.object(Path, "read_bytes", return_value=invalid_command),
                    patch(
                        "praxist.plugins.workflow_stages.research_loop.backend."
                        "experiment_scheduler._pid_start_time",
                        return_value=5,
                    ),
                    patch(
                        "praxist.plugins.workflow_stages.research_loop.backend."
                        "experiment_scheduler.time.monotonic",
                        side_effect=[0.0, 0.0, 3.0],
                    ),
                    patch(
                        "praxist.plugins.workflow_stages.research_loop.backend."
                        "experiment_scheduler.time.sleep"
                    ),
                ):
                    self.assertEqual(
                        ExperimentSchedulerService._read_ready_process(attempt_dir, "attempt-a1"),
                        {},
                    )

            fake_proc = Path(td) / "123"
            fake_proc.mkdir()
            (fake_proc / "cmdline").write_bytes(command)
            non_pid = Path(td) / "not-a-pid"
            non_pid.mkdir()
            with (
                patch.object(Path, "iterdir", return_value=[non_pid, fake_proc]),
                patch.object(os, "getpgid", return_value=456),
            ):
                self.assertEqual(
                    ExperimentSchedulerService._find_attempt_process(attempt_dir, "attempt-a1"),
                    {"pid": 123, "pgid": 456},
                )
            with patch.object(Path, "iterdir", side_effect=OSError("no proc")):
                self.assertEqual(
                    ExperimentSchedulerService._find_attempt_process(attempt_dir, "attempt-a1"),
                    {},
                )
            with patch.object(Path, "read_bytes", return_value=command):
                self.assertTrue(
                    ExperimentSchedulerService._event_process_matches(
                        {"pid": 123, "pgid": 123, "attempt_id": "attempt-a1"},
                        attempt_dir=attempt_dir,
                    )
                )

    def test_recovery_does_not_terminate_reused_process_identity(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            event = {
                "event": "launched",
                "job_id": "stale-process",
                "generation_id": 0,
                "peer_id": "gen0_peer0",
                "experiment_id": "stale-process",
                "profile": "cpu",
                "work_class": "ordinary",
                "command": ["task"],
                "environment_values": {},
                "attempt": 1,
                "pid": os.getpid(),
                "pgid": os.getpgrp(),
                "pid_start_time": -1,
                "allocation_id": "stale-allocation",
            }
            (service.state_dir / "events.jsonl").write_text(
                json.dumps(event) + "\n", encoding="utf-8"
            )
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.process_group_alive",
                    return_value=True,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.terminate_process_group"
                ) as terminate,
            ):
                service._recover_terminal_events()
            terminate.assert_not_called()
            self.assertEqual(service._jobs["stale-process"].state, "drained_unknown")

    def test_stop_retains_owner_until_active_process_is_reaped(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            first.start()
            submitted = first.submit(
                {
                    "command": [sys.executable, "-c", "import time;time.sleep(30)"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "drain-before-handoff",
                    "profile": "cpu",
                }
            )
            deadline = time.time() + 5
            while submitted.job_id not in first._active and time.time() < deadline:
                time.sleep(0.02)
            process = first._active[submitted.job_id].process
            self.assertIsNotNone(process)

            stop_thread = threading.Thread(target=first.stop)
            stop_thread.start()
            deadline = time.time() + 3
            while not first._stopping and time.time() < deadline:
                time.sleep(0.01)
            contender = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            try:
                with self.assertRaisesRegex(RuntimeError, "already owns run"):
                    contender.start()
                os.killpg(process.pid, signal.SIGTERM)
                process.wait(timeout=5)
                stop_thread.join(timeout=5)
                self.assertFalse(stop_thread.is_alive())

                successor = ExperimentSchedulerService(
                    run_dir=run_dir,
                    settings=_settings(maximum=1),
                    allocator=_GPUAllocator(""),
                )
                successor.start()
                successor.stop()
            finally:
                if process.poll() is None:
                    os.killpg(process.pid, signal.SIGKILL)
                    process.wait(timeout=5)
                stop_thread.join(timeout=5)

    def test_recovered_terminal_waiter_is_not_blocked_by_allocation_release_retry(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _GPUAllocator("")
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "recovered-release-retry",
                }
            )
            service._queue.remove(job.job_id)
            job.state = "running"
            job.pgid = 123
            allocation = Allocation(
                allocation_id="recovered-allocation",
                run_id=service.run_id,
                pid=123,
                pgid=123,
                profile="cpu",
                gpu_uuids=(),
                gpu_memory_mb=0,
                gpu_utilization_pct=0,
                started_at=time.time(),
            )
            service._active[job.job_id] = _ActiveJob(job, None, allocation, None, None)
            result: dict[str, object] = {}
            waiter = threading.Thread(
                target=lambda: result.update(service.wait(job.job_id, 2.0)),
                daemon=True,
            )
            waiter.start()
            time.sleep(0.02)
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.process_group_alive",
                    return_value=False,
                ),
                patch.object(allocator, "release", side_effect=OSError("registry unavailable")),
            ):
                service._reap()
            waiter.join(timeout=0.5)
            self.assertFalse(waiter.is_alive())
            self.assertEqual(result["job"]["state"], "drained_unknown")  # type: ignore[index]
            self.assertIn(job.job_id, service._active)

            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler.process_group_alive",
                return_value=False,
            ):
                service._reap()
            self.assertNotIn(job.job_id, service._active)
            events = [
                json.loads(line)
                for line in (service.state_dir / "events.jsonl").read_text().splitlines()
            ]
            self.assertEqual(
                sum(
                    event["event"] == "completed" and event.get("job_id") == job.job_id
                    for event in events
                ),
                1,
            )

    def test_run_dir_is_resolved_before_child_uses_different_cwd(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            previous = Path.cwd()
            try:
                os.chdir(td)
                service = ExperimentSchedulerService(
                    run_dir=Path("relative-run"), settings=_settings(maximum=1)
                )
            finally:
                os.chdir(previous)
            self.assertTrue(service.run_dir.is_absolute())
            self.assertEqual(service.run_dir, Path(td).resolve() / "relative-run")

    def test_unknown_explicit_profile_is_rejected_without_default_fallback(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            with self.assertRaisesRegex(ExperimentRejected, "unknown resource profile"):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "profile-typo",
                        "profile": "gup",
                    }
                )

    def test_scheduler_endpoint_enters_scoped_agent_runtime_environment(self) -> None:
        from praxist.plugins.workflow_stages.research_loop.backend.agent import (
            _legacy_runtime_env_keys,
        )

        self.assertIn("PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT", _legacy_runtime_env_keys())

    def test_final_launcher_overrides_inherited_gpu_zero_with_assigned_uuid(self) -> None:
        with (
            tempfile.TemporaryDirectory() as td,
            patch.dict(os.environ, {"CUDA_VISIBLE_DEVICES": "0"}, clear=False),
        ):
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=2),
                allocator=_GPUAllocator("GPU-exact-uuid"),
            )
            service.start()
            output = Path(td) / "env.json"
            try:
                code = submit_and_wait(
                    [
                        sys.executable,
                        "-c",
                        "import json,os,sys;json.dump({k:os.environ.get(k) for k in "
                        "['CUDA_VISIBLE_DEVICES','NVIDIA_VISIBLE_DEVICES','PRAXIST_ASSIGNED_GPU_UUIDS']},open(sys.argv[1],'w'))",
                        str(output),
                    ],
                    peer_id="gen0_peer0",
                    experiment_id="semantic-variant-a",
                    profile="gpu",
                    run_dir=run_dir,
                )
            finally:
                service.stop()
            self.assertEqual(code, 0)
            payload = json.loads(output.read_text())
            self.assertEqual(payload["CUDA_VISIBLE_DEVICES"], "GPU-exact-uuid")
            self.assertEqual(payload["NVIDIA_VISIBLE_DEVICES"], "GPU-exact-uuid")
            self.assertEqual(payload["PRAXIST_ASSIGNED_GPU_UUIDS"], "GPU-exact-uuid")

    def test_final_launcher_preserves_submitter_environment_and_cwd(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td).resolve() / "run"
            work_dir = Path(td).resolve() / "peer-workspace"
            work_dir.mkdir()
            output = Path(td).resolve() / "context.json"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("", limit=1),
            )
            service.start()
            previous = Path.cwd()
            try:
                os.chdir(work_dir)
                with patch.dict(os.environ, {"TASK_SCOPED_VALUE": "kept"}, clear=False):
                    code = submit_and_wait(
                        [
                            sys.executable,
                            "-c",
                            "import json,os,sys;json.dump({'cwd':os.getcwd(),"
                            "'value':os.getenv('TASK_SCOPED_VALUE')},open(sys.argv[1],'w'))",
                            str(output),
                        ],
                        peer_id="gen0_peer0",
                        experiment_id="context-preserved",
                        run_dir=run_dir,
                        cwd=Path("."),
                    )
            finally:
                os.chdir(previous)
                service.stop()
            self.assertEqual(code, 0)
            self.assertEqual(
                json.loads(output.read_text()),
                {"cwd": str(work_dir), "value": "kept"},
            )

    def test_relative_evaluator_runs_from_task_root_with_isolated_python_environment(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td).resolve()
            run_dir = root / "run"
            task_root = root / "task"
            caller_cwd = run_dir / "gen_0" / "peer_workspace"
            evaluator = task_root / "evaluations" / "public" / "run.py"
            output = root / "evaluation-context.json"
            evaluator.parent.mkdir(parents=True)
            caller_cwd.mkdir(parents=True)
            evaluator.write_text(
                "import json, os, pathlib, sys\n"
                "pathlib.Path(sys.argv[1]).write_text(json.dumps({"
                "'cwd': os.getcwd(), 'pythonpath': os.getenv('PYTHONPATH')}))\n",
                encoding="utf-8",
            )
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("", limit=1),
            )
            service.start()
            previous = Path.cwd()
            try:
                os.chdir(caller_cwd)
                with patch.dict(
                    os.environ,
                    {
                        "PRAXIST_TASK_PROJECT_PATH": str(task_root),
                        "PRAXIST_TASK_PYTHON": sys.executable,
                        "PYTHONPATH": "/runner/site-packages",
                    },
                    clear=False,
                ):
                    code = submit_and_wait(
                        ["python", "evaluations/public/run.py", str(output)],
                        peer_id="gen0_peer0",
                        experiment_id="relative-task-evaluator",
                        run_dir=run_dir,
                    )
            finally:
                os.chdir(previous)
                service.stop()

            self.assertEqual(code, 0)
            self.assertEqual(
                json.loads(output.read_text(encoding="utf-8")),
                {"cwd": str(caller_cwd), "pythonpath": None},
            )

    def test_explicit_empty_environment_does_not_inherit_scheduler_credentials(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            output = Path(td) / "environment.json"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            service.start()
            try:
                job = service.submit(
                    {
                        "command": [
                            sys.executable,
                            "-c",
                            "import json,os,sys;json.dump({'home':os.getenv('HOME'),"
                            "'secret':os.getenv('OPENAI_API_KEY')},open(sys.argv[1],'w'))",
                            str(output),
                        ],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "empty-environment",
                        "environment": {},
                    }
                )
                result = service.wait(job.job_id, 5)["job"]
            finally:
                service.stop()
            self.assertEqual(result["state"], "completed")
            self.assertEqual(json.loads(output.read_text()), {"home": None, "secret": None})

    def test_semantic_identity_deduplicates_command_spelling(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(run_dir=Path(td) / "run", settings=_settings())
            first = service.submit(
                {
                    "command": ["python", "a.py", "--output", "one"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "same-science",
                }
            )
            second = service.submit(
                {
                    "command": ["python", "a.py", "--output", "two", "--verbose"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "same-science",
                }
            )
            self.assertEqual(first.job_id, second.job_id)
            self.assertEqual(len(service._queue), 1)

    def test_close_atomically_rejects_queued_work_but_drains_running(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1, initial=1),
                allocator=_GPUAllocator("", limit=1),
            )
            service.start()
            try:
                first = service.submit(
                    {
                        "command": [sys.executable, "-c", "import time;time.sleep(2)"],
                        "peer_id": "gen2_peer0",
                        "generation_id": 2,
                        "experiment_id": "already-running",
                    }
                )
                deadline = time.time() + 3
                while service._jobs[first.job_id].state != "running" and time.time() < deadline:
                    time.sleep(0.02)
                second = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen2_peer1",
                        "generation_id": 2,
                        "experiment_id": "queued-late",
                    }
                )
                freeze_generation(2, "mature_quorum")
                second_result = service.wait(second.job_id, 2)["job"]
                first_result = service.wait(first.job_id, 5)["job"]
                self.assertEqual(second_result["state"], "rejected")
                self.assertEqual(first_result["state"], "completed")
                same = service.submit(
                    {
                        "command": [sys.executable, "-c", "raise SystemExit(99)"],
                        "peer_id": "gen2_peer0",
                        "generation_id": 2,
                        "experiment_id": "already-running",
                    }
                )
                self.assertEqual(same.job_id, first.job_id)
                with self.assertRaises(ExperimentRejected):
                    service.submit(
                        {
                            "command": [sys.executable, "-c", "pass"],
                            "peer_id": "gen2_peer2",
                            "generation_id": 2,
                            "experiment_id": "after-close",
                        }
                    )
            finally:
                service.stop()

    def test_only_exit_75_retries_and_keeps_one_semantic_job(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            marker = Path(td) / "attempt"
            code = (
                "import pathlib,sys;p=pathlib.Path(sys.argv[1]);"
                "seen=p.exists();p.write_text('x');sys.exit(0 if seen else 75)"
            )
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            service.start()
            try:
                result = submit_and_wait(
                    [sys.executable, "-c", code, str(marker)],
                    peer_id="gen0_peer0",
                    experiment_id="retry-me",
                    run_dir=run_dir,
                )
                status = service.status()
            finally:
                service.stop()
            self.assertEqual(result, 0)
            self.assertEqual(len(status["jobs"]), 1)
            self.assertEqual(status["jobs"][0]["attempts"], 2)
            self.assertEqual(status["jobs"][0]["failure_category"], "")
            self.assertEqual(status["failure_counts_by_category"], {})
            events = [
                json.loads(line)
                for line in (run_dir / "resource_scheduler" / "events.jsonl")
                .read_text()
                .splitlines()
            ]
            self.assertEqual(
                [event["event"] for event in events],
                [
                    "submitted",
                    "launch_intent",
                    "process_started",
                    "launched",
                    "retry_queued",
                    "launch_intent",
                    "process_started",
                    "launched",
                    "completed",
                    "admission_closed",
                ],
            )
            retry_event = next(event for event in events if event["event"] == "retry_queued")
            self.assertEqual(retry_event["failure_category"], "infrastructure")

    def test_corrected_terminal_request_requires_explicit_retry_and_keeps_identity(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            service.start()
            command = [
                sys.executable,
                "-c",
                "import os,sys;sys.exit(0 if os.getenv('PRAXIST_TEST_FIXED') == '1' else 9)",
            ]
            try:
                with patch.dict(os.environ, {}, clear=False):
                    os.environ.pop("PRAXIST_TEST_FIXED", None)
                    self.assertEqual(
                        submit_and_wait(
                            command,
                            peer_id="gen0_peer0",
                            experiment_id="corrected-contract",
                            run_dir=run_dir,
                        ),
                        9,
                    )

                duplicate_response = service.handle_request(
                    {
                        "action": "submit",
                        "command": command,
                        "environment": dict(os.environ),
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "corrected-contract",
                    }
                )
                self.assertTrue(duplicate_response["existing_terminal_job"])
                self.assertTrue(duplicate_response["retry_requires_explicit_request"])

                with patch.dict(os.environ, {"PRAXIST_TEST_FIXED": "1"}):
                    with self.assertRaisesRegex(ExperimentRejected, "--retry-terminal"):
                        submit_and_wait(
                            command,
                            peer_id="gen0_peer0",
                            experiment_id="corrected-contract",
                            run_dir=run_dir,
                        )
                    self.assertEqual(
                        submit_and_wait(
                            command,
                            peer_id="gen0_peer0",
                            experiment_id="corrected-contract",
                            run_dir=run_dir,
                            retry_terminal=True,
                        ),
                        0,
                    )
                status = service.status()
            finally:
                service.stop()

            self.assertEqual(len(status["jobs"]), 1)
            self.assertEqual(status["jobs"][0]["state"], "completed")
            self.assertEqual(status["jobs"][0]["attempts"], 2)
            events = [
                json.loads(line)
                for line in (run_dir / "resource_scheduler" / "events.jsonl")
                .read_text(encoding="utf-8")
                .splitlines()
            ]
            explicit_retry = next(
                event
                for event in events
                if event.get("event") == "retry_queued"
                and event.get("retry_reason") == "explicit_terminal_retry"
            )
            self.assertEqual(explicit_retry["prior_terminal_state"], "failed")
            self.assertEqual(sum(event.get("event") == "submitted" for event in events), 1)

    def test_explicit_terminal_retry_rejects_nonretryable_states_and_wrong_peer(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(run_dir=Path(td) / "run", settings=_settings())
            request = {
                "command": [sys.executable, "-c", "pass"],
                "peer_id": "gen0_peer0",
                "generation_id": 0,
                "experiment_id": "stable-science",
            }
            with self.assertRaisesRegex(ExperimentRejected, "retained failed or rejected"):
                service.submit({**request, "retry_terminal": True})
            queued = service.submit(request)
            duplicate = service.submit({**request, "retry_terminal": True})
            self.assertIs(duplicate, queued)
            with self.assertRaisesRegex(ExperimentRejected, "current state is 'queued'"):
                service.submit(
                    {
                        **request,
                        "command": [sys.executable, "-c", "raise SystemExit(2)"],
                        "retry_terminal": True,
                    }
                )

            service._queue.remove(queued.job_id)
            service._finish_without_launch(queued, "task_contract_rejected")
            with self.assertRaisesRegex(ExperimentRejected, "inherited runtime identity"):
                service.submit(
                    {
                        **request,
                        "environment": {
                            **os.environ,
                            "PRAXIST_PEER_ID": "gen0_peer1",
                            "PEER_ID": "gen0_peer1",
                        },
                        "retry_terminal": True,
                    }
                )
            with self.assertRaisesRegex(ExperimentRejected, "peer that owns"):
                service.submit(
                    {
                        **request,
                        "peer_id": "gen0_peer1",
                        "retry_terminal": True,
                    }
                )
            retried = service.submit({**request, "retry_terminal": True})
            self.assertEqual(retried.job_id, queued.job_id)
            self.assertEqual(retried.state, "queued")
            self.assertEqual(service._queue, [queued.job_id])
            self.assertIs(service.submit({**request, "retry_terminal": True}), retried)

            retried.state = "completed"
            self.assertIs(service.submit({**request, "retry_terminal": True}), retried)

    def test_admission_timeout_after_attempt_preserves_identity_and_attempt_count(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(run_dir=Path(td) / "run", settings=_settings())
            request = {
                "command": [sys.executable, "-c", "pass"],
                "peer_id": "gen0_peer0",
                "generation_id": 0,
                "experiment_id": "attempted-admission-timeout",
            }
            attempted = service.submit(request)
            attempted.attempts = 1

            self.assertTrue(service.cancel_queued(attempted.job_id)["cancelled"])
            self.assertEqual(attempted.state, "rejected")
            self.assertIs(service.submit(request), attempted)

            retried = service.submit({**request, "retry_terminal": True})
            self.assertEqual(retried.job_id, attempted.job_id)
            self.assertEqual(retried.attempts, 1)
            self.assertEqual(retried.state, "queued")

    def test_submit_response_does_not_mislabel_new_terminal_job_as_duplicate(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(run_dir=Path(td) / "run", settings=_settings())
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "fresh-terminal-response",
                }
            )
            job.state = "failed"
            with patch.object(service, "_submit", return_value=(job, False)):
                response = service.handle_request({"action": "submit"})

            self.assertFalse(response["existing_terminal_job"])
            self.assertFalse(response["retry_requires_explicit_request"])

    def test_explicit_retry_recovery_replaces_stale_terminal_runtime_fields(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            settings = _settings(maximum=1)
            settings.infrastructure_retries = 0
            first = ExperimentSchedulerService(run_dir=run_dir, settings=settings)
            request = {
                "command": [sys.executable, "-c", "pass"],
                "peer_id": "gen0_peer0",
                "generation_id": 0,
                "experiment_id": "retry-replay-reset",
            }
            failed = first.submit(request)
            first._queue.remove(failed.job_id)
            failed.attempts = 1
            failed.pid = 1234
            failed.pgid = 1234
            failed.log_path = "/old/attempt.log"
            failed.error = "old failure"
            first._retry_or_finish(failed, 9)
            self.assertEqual(failed.state, "failed")

            retried = first.submit({**request, "retry_terminal": True})
            self.assertEqual(retried.job_id, failed.job_id)

            resumed = ExperimentSchedulerService(run_dir=run_dir, settings=settings)
            resumed._recover_terminal_events()
            recovered = resumed._jobs[failed.job_id]
            self.assertEqual(recovered.state, "queued")
            self.assertEqual(recovered.pid, 0)
            self.assertEqual(recovered.pgid, 0)
            self.assertEqual(recovered.log_path, "")
            self.assertEqual(recovered.error, "")

    def test_launch_failure_retries_once_without_duplicate_queue_entries(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            service.start()
            try:
                code = submit_and_wait(
                    [str(Path(td) / "does-not-exist")],
                    peer_id="gen0_peer0",
                    experiment_id="missing-launcher",
                    run_dir=run_dir,
                )
                status = service.status()
            finally:
                service.stop()
            self.assertEqual(code, 75)
            self.assertEqual(status["queued"], 0)
            self.assertEqual(status["failed"], 1)
            self.assertEqual(status["jobs"][0]["attempts"], 2)

    def test_attempt_directory_failure_releases_reservation_without_limbo(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _GPUAllocator("")
            settings = _settings(maximum=1)
            settings.infrastructure_retries = 0
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=settings, allocator=allocator
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "mkdir-failure",
                }
            )
            with patch.object(Path, "mkdir", side_effect=OSError("disk unavailable")):
                self.assertTrue(service._launch(job))
            self.assertEqual(job.state, "failed")
            self.assertEqual(service._queue, [])
            self.assertEqual(service._active, {})
            self.assertEqual(len(allocator.released), 1)

    def test_log_open_failure_releases_reservation_without_limbo(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _GPUAllocator("")
            settings = _settings(maximum=1)
            settings.infrastructure_retries = 0
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=settings, allocator=allocator
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "log-failure",
                }
            )
            real_open = open

            def fail_log(path, *args, **kwargs):
                if str(path).endswith(".log"):
                    raise OSError("log unavailable")
                return real_open(path, *args, **kwargs)

            with patch("builtins.open", side_effect=fail_log):
                self.assertTrue(service._launch(job))
            self.assertEqual(job.state, "failed")
            self.assertEqual(service._queue, [])
            self.assertEqual(service._active, {})
            self.assertEqual(len(allocator.released), 1)

    def test_manifest_failure_never_releases_barrier_or_starts_experiment(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _GPUAllocator("")
            settings = _settings(maximum=1)
            settings.infrastructure_retries = 0
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=settings, allocator=allocator
            )
            child_pid_file = Path(td) / "child.pid"
            parent_code = "import pathlib,sys;pathlib.Path(sys.argv[1]).write_text('started')"
            job = service.submit(
                {
                    "command": [sys.executable, "-c", parent_code, str(child_pid_file)],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "manifest-failure-tree",
                }
            )

            def fail_manifest(_job) -> None:
                raise OSError("manifest unavailable")

            with patch.object(service, "_register_protected", side_effect=fail_manifest):
                self.assertTrue(service._launch(job))
            self.assertFalse(child_pid_file.exists())
            self.assertEqual(job.state, "failed")
            self.assertEqual(service._queue, [])
            self.assertEqual(service._active, {})
            self.assertEqual(len(allocator.released), 1)

    def test_binding_failure_terminates_started_process_and_releases_reservation(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _BindFailAllocator("")
            settings = _settings(maximum=1)
            settings.infrastructure_retries = 0
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=settings, allocator=allocator
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "import time;time.sleep(30)"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "binding-failure",
                }
            )
            self.assertTrue(service._launch(job))
            self.assertEqual(job.state, "failed")
            self.assertEqual(service._queue, [])
            self.assertEqual(service._active, {})
            self.assertEqual(len(allocator.released), 1)

    def test_retry_queued_event_recovers_after_scheduler_restart(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            marker = Path(td) / "recovered"
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = first.submit(
                {
                    "command": [
                        sys.executable,
                        "-c",
                        "import pathlib,sys;pathlib.Path(sys.argv[1]).write_text('ran')",
                        str(marker),
                    ],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "recover-retry",
                    "environment": {"TASK_RECOVERY_VALUE": "preserved"},
                }
            )
            job.command = [
                sys.executable,
                "-c",
                "import os,pathlib,sys;pathlib.Path(sys.argv[1]).write_text(os.environ['TASK_RECOVERY_VALUE'])",
                str(marker),
            ]
            first._queue.remove(job.job_id)
            job.attempts = 1
            first._retry_or_finish(job, 75)

            second = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            second.start()
            try:
                duplicate = second.submit(
                    {
                        "command": [sys.executable, "-c", "raise SystemExit(99)"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "recover-retry",
                    }
                )
                result = second.wait(duplicate.job_id, 5)["job"]
            finally:
                second.stop()
            self.assertEqual(result["state"], "completed")
            self.assertEqual(result["attempts"], 2)
            self.assertEqual(marker.read_text(), "preserved")

    def test_terminal_and_retry_state_wait_for_required_audit(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "durable-transition",
                }
            )
            service._queue.remove(job.job_id)
            job.state = "running"
            job.attempts = 1

            with (
                patch.object(service, "_append_event", side_effect=OSError("disk unavailable")),
                self.assertRaisesRegex(OSError, "disk unavailable"),
            ):
                service._retry_or_finish(job, 75)
            self.assertEqual(job.state, "running")
            self.assertNotIn(job.job_id, service._queue)

            service.settings.infrastructure_retries = 0
            with (
                patch.object(service, "_append_event", side_effect=OSError("disk unavailable")),
                self.assertRaisesRegex(OSError, "disk unavailable"),
            ):
                service._retry_or_finish(job, 1)
            self.assertEqual(job.state, "running")
            self.assertIsNone(job.exit_code)

    def test_assessment_does_not_retry_non_mature_work(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "assessment-retry",
                    "work_class": "ordinary",
                }
            )
            service._queue.remove(job.job_id)
            job.state = "running"
            job.attempts = 1
            service.begin_assessment(0)
            service._retry_or_finish(job, 75)
            self.assertEqual(job.state, "failed")
            self.assertNotIn(job.job_id, service._queue)

    def test_legacy_event_and_manifest_pair_recovers_live_job(self) -> None:
        if not Path("/proc").is_dir():
            self.skipTest("/proc filesystem required for process environment inspection")
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            process_env = {
                **os.environ,
                "PRAXIST_EXPERIMENT_ID": "legacy-live",
                "PRAXIST_EXPERIMENT_ATTEMPT_ID": "legacy-job-a1",
            }
            process = subprocess.Popen(
                [sys.executable, "-c", "import time;time.sleep(30)"],
                start_new_session=True,
                env=process_env,
            )
            path = protected_pids._manifest_path("gen1_peer0", run_dir)
            protected_pids._write_manifest(
                path,
                [
                    protected_pids.ProtectedEntry(
                        pid=process.pid,
                        pgid=process.pid,
                        peer_id="gen1_peer0",
                        tag="legacy-live",
                    )
                ],
            )
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            event = {
                "event": "launched",
                "job_id": "legacy-job",
                "generation_id": 1,
                "peer_id": "gen1_peer0",
                "experiment_id": "legacy-live",
                "profile": "cpu",
                "work_class": "ordinary",
                "command": [sys.executable, "-c", "pass"],
                "environment_values": {},
                "attempt": 1,
                "attempt_id": "legacy-job-a1",
                "pid": process.pid,
                "pgid": process.pid,
                "allocation_id": "legacy-allocation",
            }
            (service.state_dir / "events.jsonl").write_text(
                json.dumps(event) + "\n", encoding="utf-8"
            )
            try:
                service._recover_terminal_events()
                self.assertIn("legacy-job", service._active)
                persisted = protected_pids._read_manifest(path)
                self.assertEqual(len(persisted), 1)
                self.assertIsNotNone(persisted[0].pid_start_time)
            finally:
                os.killpg(process.pid, signal.SIGTERM)
                process.wait(timeout=5)

    def test_eventless_legacy_manifest_reserves_semantic_identity(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            process = subprocess.Popen(
                [sys.executable, "-c", "import time;time.sleep(30)"],
                start_new_session=True,
            )
            path = protected_pids._manifest_path("gen2_peer0", run_dir)
            protected_pids._write_manifest(
                path,
                [
                    protected_pids.ProtectedEntry(
                        pid=process.pid,
                        pgid=process.pid,
                        peer_id="gen2_peer0",
                        tag="eventless-legacy",
                    )
                ],
            )
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("", limit=1),
            )
            try:
                service._recover_active_process_groups()
                recovered = next(iter(service._active.values())).job
                duplicate = service.submit(
                    {
                        "command": [sys.executable, "-c", "raise SystemExit(99)"],
                        "peer_id": "gen2_peer0",
                        "generation_id": 2,
                        "experiment_id": "eventless-legacy",
                    }
                )
                self.assertEqual(duplicate.job_id, recovered.job_id)
                self.assertEqual(service._queue, [])
                unrelated = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen2_peer1",
                        "generation_id": 2,
                        "experiment_id": "unrelated-new-work",
                    }
                )
                service._launch_ready_jobs()
                self.assertEqual(unrelated.state, "queued")
                self.assertIn(unrelated.job_id, service._queue)
            finally:
                os.killpg(process.pid, signal.SIGTERM)
                process.wait(timeout=5)

    def test_manifest_recovery_keeps_descendants_after_launcher_exit(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            launcher = subprocess.Popen(
                [
                    sys.executable,
                    "-c",
                    "import subprocess,sys; "
                    "subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'], "
                    "stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)",
                ],
                start_new_session=True,
            )
            launcher_start = protected_pids._pid_start_time(launcher.pid)
            launcher.wait(timeout=5)
            path = protected_pids._manifest_path("gen4_peer0", run_dir)
            protected_pids._write_manifest(
                path,
                [
                    protected_pids.ProtectedEntry(
                        pid=launcher.pid,
                        pgid=launcher.pid,
                        pid_start_time=launcher_start,
                        peer_id="gen4_peer0",
                        tag="descendant-live",
                    )
                ],
            )
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("", limit=1),
            )
            try:
                service._recover_active_process_groups()
                self.assertEqual(len(service._active), 1)
                recovered = next(iter(service._active.values())).job
                self.assertEqual(recovered.experiment_id, "descendant-live")
                self.assertEqual(recovered.pgid, launcher.pid)
            finally:
                with contextlib.suppress(ProcessLookupError):
                    os.killpg(launcher.pid, signal.SIGKILL)

    def test_assessment_keeps_idle_waiter_for_future_maturity_debt(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            settings = _settings(maximum=2)
            settings.supply_idle_samples = 1
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=settings,
                allocator=_GPUAllocator("", limit=2),
            )
            completed = [1]
            service.open_generation(0, deadline=time.time() + 600, cohort_size=2)
            service.configure_generation_maturity(
                0,
                cohort_size=2,
                mature_target=1,
                count_callback=lambda: completed[0],
            )
            service.begin_assessment(0)
            service.register_idle_supply("gen0_peer0", 0)
            first = HostSnapshot(8, 5, 5, 0, observed_at=1.0)
            service._refresh_mature_counts(first)
            service._reconcile_supply(first)
            self.assertIn("gen0_peer0", service._idle_supply_waiters)
            self.assertEqual(service._supply_leases, {})

            completed[0] = 0
            second = HostSnapshot(8, 5, 5, 0, observed_at=2.0)
            service._refresh_mature_counts(second)
            service._reconcile_supply(second)
            self.assertNotIn("gen0_peer0", service._idle_supply_waiters)
            self.assertEqual(len(service._supply_leases), 1)

            lease_id = next(iter(service._supply_leases))
            service.release_supply_lease(lease_id, "gen0_peer0", declined=True)
            service.register_idle_supply("gen0_peer0", 0)
            third = HostSnapshot(8, 5, 5, 0, observed_at=3.0)
            service._reconcile_supply(third)
            self.assertEqual(service._supply_leases, {})

            service._declined_supply_peers[("gen0_peer0", 0, "mature")] = (0.0, 1)
            fourth = HostSnapshot(8, 5, 5, 0, observed_at=4.0)
            service._reconcile_supply(fourth)
            self.assertEqual(len(service._supply_leases), 1)

    def test_maturity_refresh_does_not_depend_on_a_new_host_sample(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("", limit=1),
            )
            completed = {"count": 0}
            service.open_generation(0, deadline=time.time() + 600, cohort_size=1)
            service.configure_generation_maturity(
                0,
                cohort_size=1,
                mature_target=1,
                count_callback=lambda: completed["count"],
            )
            snapshot = HostSnapshot(8, 5, 5, 0, observed_at=1.0)
            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler.time.monotonic",
                side_effect=(10.0, 16.0),
            ):
                service._refresh_mature_counts(snapshot)
                completed["count"] = 1
                service._refresh_mature_counts(snapshot)
            self.assertEqual(
                service.status()["resource_supply"]["maturity"]["0"]["completed"],
                1,
            )

    def test_launch_intent_recovers_waiting_barrier_without_duplicate_execution(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td).resolve() / "run"
            marker = Path(td).resolve() / "executed"
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = first.submit(
                {
                    "command": [
                        sys.executable,
                        "-c",
                        "import os,pathlib,sys; "
                        "from praxist.plugins.workflow_stages.research_loop.backend."
                        "experiment_scheduler_client import scheduler_attempt_is_active; "
                        "active=scheduler_attempt_is_active("
                        "pathlib.Path(os.environ['PRAXIST_RUN_DIR']), "
                        "os.environ['PRAXIST_EXPERIMENT_ATTEMPT_ID'], os.getpgrp()); "
                        "pathlib.Path(sys.argv[1]).write_text('active' if active else 'inactive')",
                        str(marker),
                    ],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "crash-after-popen",
                    "environment": dict(os.environ),
                }
            )
            first._queue.remove(job.job_id)
            job.attempts = 1
            attempt_id = f"{job.job_id}-a1"
            attempt_dir = first.state_dir / "attempts" / attempt_id
            attempt_dir.mkdir(parents=True)
            job.environment.update(
                {
                    "PRAXIST_RUN_DIR": str(run_dir),
                    "AUTO_RESEARCH_RUN_DIR": str(run_dir),
                    "PRAXIST_EXPERIMENT_ATTEMPT_ID": attempt_id,
                    "PRAXIST_EXPERIMENT_ATTEMPT_DIR": str(attempt_dir),
                }
            )
            ready = attempt_dir / "READY.json"
            go = attempt_dir / "GO.json"
            first._append_event(
                {
                    "event": "launch_intent",
                    **first._event_identity(job, include_request=True),
                    "allocation_id": "recovered-allocation",
                    "gpu_uuids": [],
                    "attempt_id": attempt_id,
                    "attempt_dir": str(attempt_dir),
                },
                required=True,
            )
            process = subprocess.Popen(
                [
                    sys.executable,
                    "-E",
                    "-S",
                    str(
                        Path(
                            "praxist/plugins/workflow_stages/research_loop/backend/experiment_exec.py"
                        ).resolve()
                    ),
                    str(ready),
                    str(go),
                    attempt_id,
                    *job.command,
                ],
                env={**os.environ, **job.environment},
                start_new_session=True,
            )
            deadline = time.time() + 3
            while not ready.exists() and time.time() < deadline:
                time.sleep(0.01)
            self.assertTrue(ready.exists())
            self.assertFalse(marker.exists())

            second = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            second.start()
            try:
                recovered = second.submit(
                    {
                        "command": [sys.executable, "-c", "raise SystemExit(99)"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "crash-after-popen",
                    }
                )
                process.wait(timeout=5)
                result = second.wait(recovered.job_id, 5)["job"]
            finally:
                if process.poll() is None:
                    os.killpg(process.pid, signal.SIGKILL)
                process.wait(timeout=5)
                second.stop()
            self.assertEqual(result["state"], "drained_unknown")
            self.assertEqual(second.status()["queued"], 0)
            self.assertEqual(marker.read_text(), "active")

    def test_recovery_requeues_waiting_barrier_after_task_checkout_moves(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td).resolve()
            run_dir = root / "run"
            old_task = root / "old" / "task"
            new_task = root / "new" / "task"
            old_evaluator = old_task / "evaluations" / "run.py"
            new_evaluator = new_task / "evaluations" / "run.py"
            marker = root / "executed"
            old_evaluator.parent.mkdir(parents=True)
            new_evaluator.parent.mkdir(parents=True)
            old_evaluator.write_text(
                "import pathlib,sys;pathlib.Path(sys.argv[1]).write_text('old')\n",
                encoding="utf-8",
            )
            new_evaluator.write_text(
                "import pathlib,sys;pathlib.Path(sys.argv[1]).write_text('new')\n",
                encoding="utf-8",
            )
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = first.submit(
                {
                    "command": [sys.executable, str(old_evaluator), str(marker)],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "relocated-before-go",
                    "profile": "cpu",
                    "cwd": str(old_task),
                    "environment": {"PRAXIST_TASK_PROJECT_PATH": str(old_task)},
                }
            )
            first._queue.remove(job.job_id)
            job.attempts = 1
            attempt_id = f"{job.job_id}-a1"
            attempt_dir = first.state_dir / "attempts" / attempt_id
            attempt_dir.mkdir(parents=True)
            ready = attempt_dir / "READY.json"
            go = attempt_dir / "GO.json"
            first._append_event(
                {
                    "event": "launch_intent",
                    **first._event_identity(job, include_request=True),
                    "allocation_id": "relocated-barrier-allocation",
                    "gpu_uuids": [],
                    "attempt_id": attempt_id,
                    "attempt_dir": str(attempt_dir),
                },
                required=True,
            )
            process = subprocess.Popen(
                [
                    sys.executable,
                    "-E",
                    "-S",
                    str(
                        Path(
                            "praxist/plugins/workflow_stages/research_loop/backend/experiment_exec.py"
                        ).resolve()
                    ),
                    str(ready),
                    str(go),
                    attempt_id,
                    *job.command,
                ],
                cwd=old_task,
                env={**os.environ, **job.environment},
                start_new_session=True,
            )
            deadline = time.time() + 3
            while not ready.exists() and time.time() < deadline:
                time.sleep(0.01)
            self.assertTrue(ready.exists())

            second = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            try:
                with patch.dict(
                    os.environ,
                    {"PRAXIST_TASK_PROJECT_PATH": str(new_task)},
                    clear=False,
                ):
                    second.start()
                    result = second.wait(job.job_id, 10)["job"]
                process.wait(timeout=5)
            finally:
                if process.poll() is None:
                    os.killpg(process.pid, signal.SIGKILL)
                    process.wait(timeout=5)
                second.stop()

            self.assertEqual(result["state"], "completed")
            self.assertEqual(marker.read_text(encoding="utf-8"), "new")
            events = (second.state_dir / "events.jsonl").read_text(encoding="utf-8")
            self.assertIn('"recovery_reason": "task_context_relocated_before_launch"', events)

    def test_resume_uses_latest_launched_state_instead_of_requeueing_old_retry(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = first.submit(
                {
                    "command": [sys.executable, "-c", "import time;time.sleep(30)"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "live-second-attempt",
                }
            )
            first._queue.remove(job.job_id)
            job.attempts = 1
            first._retry_or_finish(job, 75)
            self.assertTrue(first._launch(job))
            process = first._active[job.job_id].process
            assert process is not None

            second = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            second.start()
            try:
                self.assertEqual(second.status()["queued"], 0)
                self.assertEqual(second.status()["running"], 1)
                duplicate = second.submit(
                    {
                        "command": [sys.executable, "-c", "raise SystemExit(99)"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "live-second-attempt",
                    }
                )
                self.assertEqual(duplicate.semantic_key, job.semantic_key)
            finally:
                os.killpg(process.pid, signal.SIGKILL)
                process.wait(timeout=5)
                second.stop()
                first._active[job.job_id].log_handle.close()

    def test_resume_preserves_identity_when_process_finished_before_terminal_event(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            marker = Path(td) / "runs"
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = first.submit(
                {
                    "command": [
                        sys.executable,
                        "-c",
                        "import pathlib,sys;pathlib.Path(sys.argv[1]).write_text('once')",
                        str(marker),
                    ],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "finished-before-reap",
                }
            )
            self.assertTrue(first._launch(job))
            process = first._active[job.job_id].process
            assert process is not None
            process.wait(timeout=5)
            first._active[job.job_id].log_handle.close()

            second = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            second.start()
            try:
                duplicate = second.submit(
                    {
                        "command": [sys.executable, "-c", "raise SystemExit(99)"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "finished-before-reap",
                    }
                )
                recovered = second.wait(duplicate.job_id, 1)["job"]
            finally:
                second.stop()
            self.assertEqual(recovered["state"], "drained_unknown")
            self.assertEqual(marker.read_text(), "once")

    def test_deadline_admission_rejects_late_full_work_without_killing_active_work(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            service.open_generation(4, deadline=time.time() + 5)
            with self.assertRaises(ExperimentRejected):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen4_peer0",
                        "generation_id": 4,
                        "experiment_id": "full-too-late",
                        "work_class": "mature",
                        "eta_seconds": 60,
                    }
                )

    def test_assessment_uses_bounded_completion_probability_for_mature_topup(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            service.open_generation(4, deadline=time.time() + 30)
            service.begin_assessment(4, "maturity_debt")

            admitted = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen4_peer0",
                    "generation_id": 4,
                    "experiment_id": "probable-mature-topup",
                    "work_class": "mature",
                    "eta_seconds": 50,
                }
            )
            self.assertEqual(admitted.state, "queued")
            with self.assertRaises(ExperimentRejected):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen4_peer1",
                        "generation_id": 4,
                        "experiment_id": "improbable-mature-topup",
                        "work_class": "mature",
                        "eta_seconds": 60,
                    }
                )

    def test_successful_wall_time_calibrates_assessment_completion_probability(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "calibration",
                    "eta_seconds": 100,
                }
            )
            service._observe_runtime(job, started_at=time.time() - 50)
            self.assertAlmostEqual(service._runtime_log_ratios[-1], math.log(0.5), places=2)

    def test_gpu_retry_keeps_the_same_profile_and_never_falls_back_to_cpu(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            marker = Path(td) / "attempt"
            observed = Path(td) / "observed.jsonl"
            code = (
                "import json,os,pathlib,sys;"
                "p=pathlib.Path(sys.argv[1]);o=pathlib.Path(sys.argv[2]);"
                "f=o.open('a');f.write(json.dumps({'profile':os.getenv('PRAXIST_RESOURCE_PROFILE'),"
                "'gpu':os.getenv('CUDA_VISIBLE_DEVICES')})+'\\n');f.close();"
                "seen=p.exists();p.write_text('x');sys.exit(0 if seen else 75)"
            )
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("GPU-fixed"),
            )
            service.start()
            try:
                result = submit_and_wait(
                    [sys.executable, "-c", code, str(marker), str(observed)],
                    peer_id="gen0_peer0",
                    experiment_id="gpu-stable-retry",
                    profile="gpu",
                    run_dir=run_dir,
                )
            finally:
                service.stop()
            rows = [json.loads(line) for line in observed.read_text().splitlines()]
            self.assertEqual(result, 0)
            self.assertEqual(rows, [{"profile": "gpu", "gpu": "GPU-fixed"}] * 2)

    def test_protected_pid_launch_is_a_facade_in_central_mode(self) -> None:
        with (
            patch.dict(os.environ, {"PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT": "/tmp/fake"}),
            patch(
                "praxist.plugins.workflow_stages.research_loop.backend.experiment_scheduler_client.submit_and_wait",
                return_value=0,
            ) as submit,
            patch.object(protected_pids.subprocess, "Popen") as popen,
        ):
            result = protected_pids.launch_command(
                ["python", "train.py"],
                peer_id="gen0_peer0",
                tag="variant-a",
                resource_profile="gpu",
                work_class="mature",
                retry_terminal=True,
            )
        self.assertEqual(result, 0)
        submit.assert_called_once()
        self.assertTrue(submit.call_args.kwargs["retry_terminal"])
        popen.assert_not_called()

    def test_protected_pid_terminal_retry_uses_inherited_peer_identity(self) -> None:
        with (
            patch.dict(
                os.environ,
                {
                    "PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT": "/tmp/fake",
                    "PRAXIST_PEER_ID": "gen0_peer1",
                    "PEER_ID": "gen0_peer1",
                },
            ),
            patch(
                "praxist.plugins.workflow_stages.research_loop.backend.experiment_scheduler_client.submit_and_wait",
            ) as submit,
            self.assertRaisesRegex(ValueError, "inherited runtime identity"),
        ):
            protected_pids.launch_command(
                ["python", "train.py"],
                peer_id="gen0_peer0",
                tag="variant-a",
                retry_terminal=True,
            )
        submit.assert_not_called()

    def test_protected_pid_identity_uses_ps_fallback_without_proc(self) -> None:
        with (
            patch.object(Path, "read_text", side_effect=OSError("no procfs")),
            patch("praxist.cli.registry.process_start_token", return_value="ps:stable-start"),
        ):
            observed = protected_pids._pid_start_time(4321)

        self.assertEqual(observed, "ps:stable-start")
        entry = protected_pids.ProtectedEntry(pid=4321, pid_start_time=observed)
        with patch.object(
            protected_pids,
            "_pid_start_time",
            return_value="ps:stable-start",
        ):
            self.assertTrue(protected_pids._entry_process_identity_matches(entry))
        with patch.object(protected_pids, "_pid_start_time", return_value=None):
            self.assertFalse(protected_pids._entry_process_identity_matches(entry))

    def test_run_owned_endpoint_prevents_peer_from_downgrading_to_legacy(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(run_dir=run_dir, settings=_settings(maximum=1))
            scheduler_dir = run_dir / "resource_scheduler"
            scheduler_dir.mkdir(parents=True, exist_ok=True)
            (scheduler_dir / "endpoint.json").write_text(
                json.dumps({"endpoint": str(service.endpoint)}),
                encoding="utf-8",
            )
            with (
                patch.dict(
                    os.environ,
                    {
                        "PRAXIST_EXPERIMENT_SCHEDULER_CONFIG": '{"mode":"legacy"}',
                        "PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT": "",
                        "PRAXIST_EXPERIMENT_ATTEMPT_ID": "forged-attempt",
                        "PRAXIST_EXPERIMENT_ATTEMPT_DIR": str(
                            scheduler_dir / "attempts" / "forged-attempt"
                        ),
                    },
                    clear=False,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler_client.submit_and_wait",
                    return_value=0,
                ) as submit,
                patch.object(protected_pids.subprocess, "Popen") as popen,
            ):
                result = protected_pids.launch_command(
                    ["python", "train.py"],
                    peer_id="gen4_peer5",
                    tag="variant-e3",
                    run_dir=run_dir,
                )

        self.assertEqual(result, 0)
        self.assertEqual(
            submit.call_args.kwargs["scheduler_endpoint"],
            str(service.endpoint),
        )
        popen.assert_not_called()

    def test_run_owned_endpoint_rejects_redirected_metadata(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            scheduler_dir = run_dir / "resource_scheduler"
            scheduler_dir.mkdir(parents=True, exist_ok=True)
            (scheduler_dir / "endpoint.json").write_text(
                json.dumps({"endpoint": "/tmp/redirected.sock"}),
                encoding="utf-8",
            )
            with (
                patch.dict(
                    os.environ,
                    {
                        "PROTECTED_PIDS_DIR": str(run_dir / "protected_pids"),
                        "PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT": "",
                    },
                    clear=False,
                ),
                patch.object(protected_pids.subprocess, "Popen") as popen,
                self.assertRaisesRegex(SchedulerUnavailable, "does not match its run"),
            ):
                protected_pids.launch_command(
                    ["python", "train.py"],
                    peer_id="gen0_peer0",
                    tag="redirected-endpoint",
                    run_dir=run_dir,
                )
        popen.assert_not_called()

    def test_explicit_run_dir_cannot_cross_inherited_run_boundary(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            inherited_run = Path(td) / "run-a"
            explicit_run = Path(td) / "run-b"
            with (
                patch.dict(
                    os.environ,
                    {
                        "PRAXIST_RUN_DIR": str(inherited_run),
                        "AUTO_RESEARCH_RUN_DIR": str(inherited_run),
                        "PROTECTED_PIDS_DIR": str(explicit_run / "protected_pids"),
                    },
                    clear=False,
                ),
                patch.object(protected_pids.subprocess, "Popen") as popen,
                self.assertRaisesRegex(ValueError, "does not match the inherited run"),
            ):
                protected_pids.launch_command(
                    ["python", "train.py"],
                    peer_id="gen0_peer0",
                    tag="wrong-run",
                    run_dir=explicit_run,
                )
        popen.assert_not_called()

    def test_attempt_validation_rejects_inconsistent_run_evidence(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            run_dir = root / "run"
            other_run = root / "other-run"
            with (
                patch.dict(
                    os.environ,
                    {
                        "PRAXIST_RUN_DIR": str(run_dir),
                        "AUTO_RESEARCH_RUN_DIR": str(other_run),
                    },
                    clear=False,
                ),
                self.assertRaisesRegex(ValueError, "run boundaries disagree"),
            ):
                protected_pids._run_dir_from_runtime_env()

            attempt_id = "attempt-1"
            attempts_root = run_dir / "resource_scheduler" / "attempts"
            attempt_dir = attempts_root / attempt_id
            attempt_dir.mkdir(parents=True)
            environment = {
                "PRAXIST_EXPERIMENT_ATTEMPT_ID": attempt_id,
                "PRAXIST_EXPERIMENT_ATTEMPT_DIR": str(root / "outside" / attempt_id),
            }
            with patch.dict(os.environ, environment, clear=False):
                self.assertFalse(protected_pids._inside_active_scheduler_attempt(run_dir))

            environment["PRAXIST_EXPERIMENT_ATTEMPT_DIR"] = str(attempt_dir)
            (attempt_dir / "READY.json").write_text(
                json.dumps({"attempt_id": "other-attempt", "pgid": os.getpgrp()}),
                encoding="utf-8",
            )
            with patch.dict(os.environ, environment, clear=False):
                self.assertFalse(protected_pids._inside_active_scheduler_attempt(run_dir))

            (attempt_dir / "READY.json").write_text(
                json.dumps({"attempt_id": attempt_id, "pgid": os.getpgrp()}),
                encoding="utf-8",
            )
            with patch.dict(os.environ, environment, clear=False):
                self.assertFalse(protected_pids._inside_active_scheduler_attempt(run_dir))

            with patch.object(Path, "exists", side_effect=OSError("unavailable")):
                self.assertIsNone(protected_pids._run_shutdown_signal(run_dir))

    def test_forged_attempt_files_cannot_bypass_live_scheduler_authority(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(run_dir=run_dir, settings=_settings(maximum=1))
            scheduler_dir = run_dir / "resource_scheduler"
            scheduler_dir.mkdir(parents=True, exist_ok=True)
            (scheduler_dir / "endpoint.json").write_text(
                json.dumps({"endpoint": str(service.endpoint)}),
                encoding="utf-8",
            )
            attempt_dir = _write_active_attempt(run_dir, "forged-attempt")
            with (
                patch.dict(
                    os.environ,
                    {
                        "PRAXIST_EXPERIMENT_ATTEMPT_ID": "forged-attempt",
                        "PRAXIST_EXPERIMENT_ATTEMPT_DIR": str(attempt_dir),
                    },
                    clear=False,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler_client.scheduler_attempt_is_active",
                    return_value=False,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler_client.submit_and_wait",
                    return_value=0,
                ) as submit,
                patch.object(protected_pids.subprocess, "run") as direct_run,
            ):
                result = protected_pids.launch_command(
                    ["python", "train.py"],
                    peer_id="gen0_peer0",
                    tag="forged-nested",
                    run_dir=run_dir,
                )

        self.assertEqual(result, 0)
        submit.assert_called_once()
        direct_run.assert_not_called()

    def test_legacy_capacity_wait_rechecks_run_shutdown_before_launch(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            run_dir.mkdir()
            shutdown = run_dir / "ORCHESTRATOR_SHUTDOWN"

            def write_shutdown(_seconds: float) -> None:
                shutdown.write_text("source=praxist_stop\n", encoding="utf-8")

            with (
                patch.dict(
                    os.environ,
                    {
                        "PRAXIST_EXPERIMENT_SCHEDULER_CONFIG": '{"mode":"legacy"}',
                        "PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT": "",
                        "PRAXIST_EXPERIMENT_ATTEMPT_ID": "",
                    },
                    clear=False,
                ),
                patch.object(
                    protected_pids,
                    "_manifest_lock",
                    return_value=contextlib.nullcontext(),
                ),
                patch.object(protected_pids, "_read_manifest", return_value=[]),
                patch.object(
                    protected_pids,
                    "_check_peer_capacity",
                    side_effect=protected_pids.ProtectedPidCapacityError("busy"),
                ),
                patch.object(protected_pids.time, "sleep", side_effect=write_shutdown),
                patch.object(protected_pids.subprocess, "Popen") as popen,
                self.assertRaises(protected_pids.GenerationClosingLaunchError),
            ):
                protected_pids.launch_command(
                    ["python", "train.py"],
                    peer_id="gen0_peer0",
                    tag="late",
                    run_dir=run_dir,
                    wait_timeout_seconds=10,
                )

        popen.assert_not_called()

    def test_central_launch_never_falls_back_when_endpoint_is_missing(self) -> None:
        with (
            patch.dict(
                os.environ,
                {
                    "PRAXIST_EXPERIMENT_SCHEDULER_CONFIG": '{"mode":"central"}',
                    "PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT": "",
                },
                clear=False,
            ),
            patch.object(protected_pids.subprocess, "Popen") as popen,
            self.assertRaisesRegex(SchedulerUnavailable, "endpoint is unavailable"),
        ):
            protected_pids.launch_command(
                ["python", "train.py"],
                peer_id="gen0_peer0",
                tag="variant-a",
            )
        popen.assert_not_called()

        with (
            patch.dict(
                os.environ,
                {
                    "PRAXIST_EXPERIMENT_SCHEDULER_CONFIG": '{"mode":"  "}',
                    "PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT": "",
                },
                clear=False,
            ),
            patch.object(protected_pids.subprocess, "Popen") as blank_mode_popen,
            self.assertRaisesRegex(SchedulerUnavailable, "endpoint is unavailable"),
        ):
            protected_pids.launch_command(
                ["python", "train.py"],
                peer_id="gen0_peer0",
                tag="variant-b",
            )
        blank_mode_popen.assert_not_called()

    def test_scheduler_validates_only_its_live_attempt_process_group(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "attempt-authority",
                }
            )
            job.attempts = 2
            job.pgid = 4321
            job.pid = 4321
            job.state = "running"
            process = MagicMock()
            process.pid = 4321
            process.poll.return_value = None
            attempt_dir = service.state_dir / "attempts" / f"{job.job_id}-a2"
            attempt_dir.mkdir(parents=True)
            service._active[job.job_id] = _ActiveJob(
                job,
                process,
                None,
                None,
                attempt_dir,
                12345,
            )

            with (
                patch.object(protected_pids, "_pid_start_time", return_value=12345),
                patch.object(os, "getpgid", return_value=4321),
            ):
                accepted = service.handle_request(
                    {
                        "action": "validate_attempt",
                        "attempt_id": f"{job.job_id}-a2",
                        "pgid": 4321,
                    }
                )
                wrong_attempt = service.handle_request(
                    {
                        "action": "validate_attempt",
                        "attempt_id": f"{job.job_id}-a1",
                        "pgid": 4321,
                    }
                )
                wrong_group = service.handle_request(
                    {
                        "action": "validate_attempt",
                        "attempt_id": f"{job.job_id}-a2",
                        "pgid": 9876,
                    }
                )
                active_groups = service.handle_request({"action": "active_process_groups"})
                service._active[job.job_id].pid_start_time = 54321
                reused_identity = service.handle_request(
                    {
                        "action": "validate_attempt",
                        "attempt_id": f"{job.job_id}-a2",
                        "pgid": 4321,
                    }
                )
                service._active[job.job_id].pid_start_time = 12345
                process.poll.return_value = 0
                stale_attempt = service.handle_request(
                    {
                        "action": "validate_attempt",
                        "attempt_id": f"{job.job_id}-a2",
                        "pgid": 4321,
                    }
                )

        self.assertTrue(accepted["active"])
        self.assertFalse(wrong_attempt["active"])
        self.assertFalse(wrong_group["active"])
        self.assertFalse(reused_identity["active"])
        self.assertFalse(stale_attempt["active"])
        self.assertEqual(
            active_groups["groups"],
            [{"pgid": 4321, "pid": 4321, "pid_start_time": 12345}],
        )

    def test_scheduler_does_not_publish_exited_launcher_as_group_authority(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "stale-group-authority",
                }
            )
            job.pid = job.pgid = 4321
            job.state = "running"
            process = MagicMock()
            process.pid = 4321
            process.poll.return_value = 0
            service._active[job.job_id] = _ActiveJob(job, process, None, None, None)

            active_groups = service.handle_request({"action": "active_process_groups"})

        self.assertEqual(active_groups["groups"], [])

    def test_scheduler_validates_verified_recovered_attempt_without_popen(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "recovered-attempt-authority",
                }
            )
            process = subprocess.Popen(
                [sys.executable, "-c", "import time;time.sleep(30)"],
                start_new_session=True,
            )
            job.attempts = 1
            job.pid = job.pgid = process.pid
            job.state = "running"
            attempt_id = f"{job.job_id}-a1"
            attempt_dir = run_dir / "resource_scheduler" / "attempts" / attempt_id
            attempt_dir.mkdir(parents=True)
            service._active[job.job_id] = _ActiveJob(
                job,
                None,
                None,
                None,
                attempt_dir,
                protected_pids._pid_start_time(process.pid),
            )
            try:
                self.assertTrue(service.attempt_is_active(attempt_id, process.pid))
                self.assertFalse(service.attempt_is_active(f"{job.job_id}-a2", process.pid))
            finally:
                os.killpg(process.pid, signal.SIGTERM)
                process.wait(timeout=5)
            self.assertFalse(service.attempt_is_active(attempt_id, process.pid))

    def test_scheduler_validates_live_descendant_after_launcher_exit(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            child_pid_path = root / "child.pid"
            launcher = subprocess.Popen(
                [
                    sys.executable,
                    "-c",
                    "import pathlib,subprocess,sys; "
                    "child=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)']); "
                    "pathlib.Path(sys.argv[1]).write_text(str(child.pid))",
                    str(child_pid_path),
                ],
                start_new_session=True,
            )
            launcher_start = protected_pids._pid_start_time(launcher.pid)
            launcher.wait(timeout=5)
            self.assertGreater(int(child_pid_path.read_text(encoding="utf-8")), 1)
            run_dir = root / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "descendant-attempt-authority",
                }
            )
            job.attempts = 1
            job.pid = job.pgid = launcher.pid
            job.state = "running"
            attempt_id = f"{job.job_id}-a1"
            attempt_dir = run_dir / "resource_scheduler" / "attempts" / attempt_id
            attempt_dir.mkdir(parents=True)
            service._active[job.job_id] = _ActiveJob(
                job,
                launcher,
                None,
                None,
                attempt_dir,
                launcher_start,
            )
            try:
                self.assertTrue(service.attempt_is_active(attempt_id, launcher.pid))
                service._active[job.job_id].process = None
                self.assertTrue(service.attempt_is_active(attempt_id, launcher.pid))
            finally:
                os.killpg(launcher.pid, signal.SIGKILL)
                deadline = time.monotonic() + 5
                while process_group_alive(launcher.pid) and time.monotonic() < deadline:
                    time.sleep(0.02)
            self.assertFalse(service.attempt_is_active(attempt_id, launcher.pid))

    def test_nested_task_child_stays_inside_existing_scheduler_allocation(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            attempt_dir = _write_active_attempt(run_dir, "attempt-1")
            output = Path(td) / "nested"
            with (
                patch.dict(
                    os.environ,
                    {
                        "PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT": "",
                        "PRAXIST_EXPERIMENT_ATTEMPT_ID": "attempt-1",
                        "PRAXIST_EXPERIMENT_ATTEMPT_DIR": str(attempt_dir),
                    },
                    clear=False,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler_client.submit_and_wait"
                ) as submit,
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler_client.scheduler_attempt_is_active",
                    return_value=True,
                ),
            ):
                code = protected_pids.launch_command(
                    [
                        sys.executable,
                        "-c",
                        "import pathlib,sys;pathlib.Path(sys.argv[1]).write_text('ok')",
                        str(output),
                    ],
                    peer_id="gen0_peer0",
                    tag="nested",
                    run_dir=run_dir,
                )
            observed = output.read_text()
        self.assertEqual(code, 0)
        self.assertEqual(observed, "ok")
        submit.assert_not_called()

    def test_nested_task_child_reuses_the_shared_task_runtime_boundary(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td).resolve()
            task_root = root / "task"
            run_dir = root / "run"
            caller_cwd = run_dir / "attempt"
            attempt_dir = _write_active_attempt(run_dir, "attempt-1")
            evaluator = task_root / "evaluations" / "run.py"
            output = root / "nested-context.json"
            evaluator.parent.mkdir(parents=True)
            caller_cwd.mkdir(parents=True)
            evaluator.write_text(
                "import json, os, pathlib, sys\n"
                "pathlib.Path(sys.argv[1]).write_text(json.dumps({"
                "'cwd': os.getcwd(), 'pythonpath': os.getenv('PYTHONPATH'), "
                "'pythonhome': os.getenv('PYTHONHOME')}))\n",
                encoding="utf-8",
            )
            with (
                patch.dict(
                    os.environ,
                    {
                        "PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT": "/tmp/not-contacted",
                        "PRAXIST_EXPERIMENT_ATTEMPT_ID": "attempt-1",
                        "PRAXIST_EXPERIMENT_ATTEMPT_DIR": str(attempt_dir),
                        "PRAXIST_TASK_PROJECT_PATH": str(task_root),
                        "PRAXIST_TASK_PYTHON": sys.executable,
                        "PYTHONPATH": "/runner/python313",
                        "PYTHONHOME": "/runner/python313",
                    },
                    clear=True,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler_client.submit_and_wait"
                ) as submit,
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler_client.scheduler_attempt_is_active",
                    return_value=True,
                ),
            ):
                code = protected_pids.launch_command(
                    ["bash", "-lc", f"python3.11 evaluations/run.py {output}"],
                    peer_id="gen0_peer0",
                    tag="nested-task-boundary",
                    run_dir=run_dir,
                    cwd=caller_cwd,
                )

            observed = json.loads(output.read_text(encoding="utf-8"))

        self.assertEqual(code, 0)
        self.assertEqual(
            observed,
            {"cwd": str(caller_cwd), "pythonpath": None, "pythonhome": None},
        )
        submit.assert_not_called()

    def test_filesystem_close_signal_is_scheduler_fallback(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            gen_dir = run_dir / "gen_5"
            gen_dir.mkdir(parents=True)
            (gen_dir / "CLOSING_SIGNAL").write_text("closing")
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("", limit=1),
            )
            with self.assertRaises(ExperimentRejected):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen5_peer0",
                        "generation_id": 5,
                        "experiment_id": "after-signal",
                    }
                )

    def test_resume_adopts_live_process_group_and_deduplicates_it(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            process = subprocess.Popen(
                [sys.executable, "-c", "import time;time.sleep(30)"],
                start_new_session=True,
            )
            path = protected_pids._manifest_path("gen3_peer2", run_dir)
            protected_pids._write_manifest(
                path,
                [
                    protected_pids.ProtectedEntry(
                        pid=process.pid,
                        pgid=process.pid,
                        peer_id="gen3_peer2",
                        tag="same-science",
                        pid_start_time=protected_pids._pid_start_time(process.pid),
                    )
                ],
            )
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            service.start()
            try:
                recovered = next(iter(service._jobs.values()))
                duplicate = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen3_peer2",
                        "generation_id": 3,
                        "experiment_id": "same-science",
                    }
                )
                self.assertEqual(duplicate.job_id, recovered.job_id)
                os.killpg(process.pid, signal.SIGTERM)
                process.wait(timeout=5)
                result = service.wait(recovered.job_id, 5)["job"]
                self.assertEqual(result["state"], "drained_unknown")
            finally:
                if process.poll() is None:
                    os.killpg(process.pid, signal.SIGKILL)
                    process.wait(timeout=5)
                service.stop()

            second_service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            second_service.start()
            try:
                duplicate = second_service.submit(
                    {
                        "command": [sys.executable, "-c", "raise SystemExit(99)"],
                        "peer_id": "gen3_peer2",
                        "generation_id": 3,
                        "experiment_id": "same-science",
                    }
                )
                persisted = second_service.wait(duplicate.job_id, 1)["job"]
            finally:
                second_service.stop()
            self.assertEqual(persisted["state"], "drained_unknown")
            self.assertIsNone(persisted["exit_code"])

    def test_manifest_adoption_survives_repeated_restart_without_retry(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            process = subprocess.Popen(
                [sys.executable, "-c", "import time;time.sleep(30)"],
                start_new_session=True,
            )
            path = protected_pids._manifest_path("gen2_peer1", run_dir)
            protected_pids._write_manifest(
                path,
                [
                    protected_pids.ProtectedEntry(
                        pid=process.pid,
                        pgid=process.pid,
                        peer_id="gen2_peer1",
                        tag="stable-identity",
                        pid_start_time=protected_pids._pid_start_time(process.pid),
                    )
                ],
            )
            first_allocator = _GPUAllocator("")
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=first_allocator,
            )
            first._acquire_owner_lock()
            try:
                first._recover_active_process_groups()
                adopted = next(iter(first._active.values()))
                self.assertIsNotNone(adopted.allocation)
                self.assertEqual(adopted.job.profile, "cpu")
            finally:
                first._release_owner_lock()

            second_allocator = _GPUAllocator("")
            second = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=second_allocator,
            )
            try:
                with (
                    patch.dict(os.environ, {"PRAXIST_TASK_PYTHON": sys.executable}),
                    patch(
                        "praxist.plugins.workflow_stages.research_loop.backend."
                        "experiment_scheduler.terminate_process_group"
                    ) as terminate,
                ):
                    second.start()
                terminate.assert_not_called()
                recovered = next(iter(second._active.values()))
                self.assertIsNotNone(recovered.allocation)
                self.assertIn(recovered.allocation.allocation_id, second_allocator.active)
                events = (second.state_dir / "events.jsonl").read_text(encoding="utf-8")
                self.assertIn('"recovery_reason": "protected_manifest"', events)

                os.killpg(process.pid, signal.SIGTERM)
                process.wait(timeout=5)
                result = second.wait(recovered.job.job_id, 5)["job"]
                self.assertEqual(result["state"], "drained_unknown")
            finally:
                if process.poll() is None:
                    os.killpg(process.pid, signal.SIGKILL)
                    process.wait(timeout=5)
                second.stop()

    def test_event_and_manifest_recovery_share_one_host_allocation(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            root = Path(td)
            run_dir = root / "run"
            registry = HostAllocationRegistry(root / "host-allocations.json")
            settings = _settings(maximum=1)
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=ResourceAllocator(
                    settings,
                    observer=_StaticObserver(),
                    registry=registry,
                ),
            )
            process = subprocess.Popen(
                [sys.executable, "-c", "import time;time.sleep(30)"],
                start_new_session=True,
            )
            try:
                submitted = first.submit(
                    {
                        "command": [sys.executable, "-c", "import time;time.sleep(30)"],
                        "peer_id": "gen1_peer0",
                        "generation_id": 1,
                        "experiment_id": "single-allocation",
                        "profile": "cpu",
                    }
                )
                submitted.attempts = 1
                submitted.state = "running"
                submitted.pid = process.pid
                submitted.pgid = process.pid
                allocation_id = f"{first.resource_owner_id}:{submitted.job_id}:1"
                allocation = first.allocator.recover_allocation(
                    allocation_id=allocation_id,
                    run_id=first.resource_owner_id,
                    pid=process.pid,
                    pgid=process.pid,
                    profile=first.settings.profiles["cpu"],
                    gpu_uuids=(),
                    require_admission=False,
                )
                self.assertIsNotNone(allocation)
                first._register_protected(submitted)
                first._append_event(
                    {
                        "event": "launched",
                        **first._event_identity(submitted, include_request=True),
                        "pid": process.pid,
                        "pgid": process.pid,
                        "gpu_uuids": [],
                        "allocation_id": allocation_id,
                    },
                    required=True,
                )

                second = ExperimentSchedulerService(
                    run_dir=run_dir,
                    settings=settings,
                    allocator=ResourceAllocator(
                        settings,
                        observer=_StaticObserver(),
                        registry=registry,
                    ),
                )
                second.start()
                with registry.locked() as rows:
                    allocations = [row for row in rows if row.get("record_type") == "allocation"]
                self.assertEqual(len(allocations), 1)
                self.assertEqual(allocations[0]["pgid"], process.pid)
                recovered_job_id = next(iter(second._active))

                os.killpg(process.pid, signal.SIGTERM)
                process.wait(timeout=5)
                result = second.wait(recovered_job_id, 5)["job"]
                self.assertEqual(result["state"], "drained_unknown")
            finally:
                if process.poll() is None:
                    os.killpg(process.pid, signal.SIGKILL)
                    process.wait(timeout=5)
                if "second" in locals():
                    second.stop()

    def test_resume_restores_terminal_semantic_identity_without_reexecution(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            marker = Path(td) / "runs"
            code = "import pathlib,sys;p=pathlib.Path(sys.argv[1]);p.write_text(p.read_text()+'x' if p.exists() else 'x')"
            first_service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            first_service.start()
            try:
                self.assertEqual(
                    submit_and_wait(
                        [sys.executable, "-c", code, str(marker)],
                        peer_id="gen1_peer0",
                        experiment_id="terminal-science",
                        run_dir=run_dir,
                    ),
                    0,
                )
            finally:
                first_service.stop()
            second_service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            second_service.start()
            try:
                self.assertEqual(
                    submit_and_wait(
                        [sys.executable, "-c", "raise SystemExit(99)"],
                        peer_id="gen1_peer0",
                        experiment_id="terminal-science",
                        run_dir=run_dir,
                    ),
                    0,
                )
            finally:
                second_service.stop()
            self.assertEqual(marker.read_text(), "x")

    def test_idle_capacity_issues_only_directed_profile_aware_leases(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            settings = _settings(maximum=4)
            settings.supply_idle_samples = 2
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=_SupplyAllocator((("cpu", "gpu"), ("cpu",), ("cpu",))),
            )
            service.open_generation(0, deadline=time.time() + 600)
            for peer_index in range(6):
                service.register_idle_supply(f"gen0_peer{peer_index}", 0)
            first = HostSnapshot(16, 5, 5, 0, observed_at=1.0)
            second = HostSnapshot(16, 5, 5, 0, observed_at=2.0)
            service._reconcile_supply(first)
            self.assertEqual(service.status()["resource_supply"]["leases"], [])
            service._reconcile_supply(second)

            supply = service.status()["resource_supply"]
            self.assertEqual(len(supply["leases"]), 3)
            self.assertEqual(supply["idle_waiters"], 3)
            self.assertEqual(supply["leases"][0]["generation_id"], 0)
            self.assertEqual(supply["leases"][0]["admissible_profiles"], ("cpu", "gpu"))
            self.assertNotIn("gpu_uuids", supply["leases"][0])
            paths = sorted((run_dir / "gen_0" / "resource_supply").glob("*.json"))
            self.assertEqual(len(paths), 3)
            self.assertEqual(
                {path.stem for path in paths},
                {"gen0_peer0", "gen0_peer1", "gen0_peer2"},
            )
            locator = json.loads(paths[0].read_text())
            self.assertNotIn("admissible_profiles", locator)
            canonical = service.get_supply_lease(
                locator["peer_id"], locator["generation_id"], locator["lease_id"]
            )
            self.assertIn("admissible_profiles", canonical)

    def test_supply_registration_rejects_noncanonical_or_oversized_peer_ids(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.open_generation(0, deadline=time.time() + 600)
            for peer_id in ("../peer", "gen1_peer0", "gen0_peer" + "9" * 200):
                self.assertEqual(service.register_idle_supply(peer_id, 0), {})
            self.assertEqual(service.status()["resource_supply"]["idle_waiters"], 0)

    def test_supply_cleanup_and_write_do_not_follow_symlink_directory(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            external = Path(td) / "external"
            external.mkdir()
            victim = external / "victim.json"
            victim.write_text("keep")
            supply_dir = run_dir / "gen_0" / "resource_supply"
            supply_dir.parent.mkdir(parents=True)
            supply_dir.symlink_to(external, target_is_directory=True)
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 2
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service._remove_stale_supply_files()
            self.assertEqual(victim.read_text(), "keep")
            service.open_generation(0, deadline=time.time() + 600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            with self.assertRaises(OSError):
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            self.assertEqual(victim.read_text(), "keep")

    def test_supply_owner_identity_uses_absolute_run_path(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            first = ExperimentSchedulerService(
                run_dir=Path(td) / "task-a" / "run",
                settings=_settings(maximum=1),
            )
            second = ExperimentSchedulerService(
                run_dir=Path(td) / "task-b" / "run",
                settings=_settings(maximum=1),
            )
            self.assertEqual(first.run_id, second.run_id)
            self.assertNotEqual(first.supply_owner_id, second.supply_owner_id)

    def test_supply_lease_ids_are_unique_across_run_services(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 2
            services = [
                ExperimentSchedulerService(
                    run_dir=Path(td) / run_name,
                    settings=settings,
                    allocator=_SupplyAllocator((("cpu",),)),
                )
                for run_name in ("run-a", "run-b")
            ]
            lease_ids: list[str] = []
            for index, service in enumerate(services):
                service.open_generation(0, deadline=time.time() + 600)
                service.register_idle_supply(f"gen0_peer{index}", 0)
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
                lease_ids.append(service.status()["resource_supply"]["leases"][0]["lease_id"])
            self.assertEqual(len(set(lease_ids)), 2)

    def test_supply_lease_is_consumed_by_one_new_semantic_experiment(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 2
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.open_generation(0, deadline=time.time() + 600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            lease = service.status()["resource_supply"]["leases"][0]
            path = run_dir / "gen_0" / "resource_supply" / "gen0_peer0.json"
            self.assertTrue(path.exists())

            service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "planned-follow-up",
                    "supply_lease_id": lease["lease_id"],
                }
            )
            self.assertEqual(service.status()["resource_supply"]["leases"], [])
            self.assertFalse(path.exists())
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["consumed"], 1)
            self.assertEqual(stats["conversion_rate"], 1.0)
            self.assertEqual(stats["by_priority"]["frontier_followup"]["conversion_rate"], 1.0)
            second = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "ordinary-second-follow-up",
                    "supply_lease_id": lease["lease_id"],
                }
            )
            self.assertEqual(second.supply_claim_id, "")
            self.assertEqual(service.status()["resource_supply"]["stats"]["reuse_ignored"], 1)
            events = [
                json.loads(line)
                for line in (service.state_dir / "events.jsonl").read_text().splitlines()
            ]
            supply_events = [event["event"] for event in events if "supply" in event["event"]]
            self.assertEqual(
                supply_events,
                ["supply_granted", "supply_consumed", "supply_reuse_ignored"],
            )

    def test_expired_lease_submission_is_stale_not_consumed_or_reused(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 2
            service.open_generation(0, deadline=time.time() + 3600, cohort_size=1)
            service.configure_generation_maturity(
                0,
                cohort_size=1,
                mature_target=1,
                count_callback=lambda: 0,
            )
            service._refresh_mature_counts(HostSnapshot(8, 5, 5, 0, observed_at=0.5))
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            lease = service.status()["resource_supply"]["leases"][0]

            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler.time.time",
                return_value=float(lease["expires_at"]) + 1.0,
            ):
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=3.0))

            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "late-existing-plan",
                    "work_class": "mature",
                    "supply_lease_id": lease["lease_id"],
                }
            )
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(job.supply_claim_id, "")
            self.assertEqual(stats["consumed"], 0)
            self.assertEqual(stats["reuse_ignored"], 0)
            self.assertEqual(stats["stale_submission"], 1)
            self.assertEqual(stats["expired"], 1)
            self.assertEqual(stats["by_priority"]["mature"]["conversion_rate"], 0.0)

    def test_expired_active_lease_submission_is_stale_and_expires_atomically(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 2
            service.open_generation(0, deadline=time.time() + 3600, cohort_size=1)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            lease = service.status()["resource_supply"]["leases"][0]

            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler.time.time",
                return_value=float(lease["expires_at"]) + 1.0,
            ):
                job = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "late-existing-plan",
                        "supply_lease_id": lease["lease_id"],
                    }
                )

            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(job.supply_claim_id, "")
            self.assertEqual(stats["consumed"], 0)
            self.assertEqual(stats["expired"], 1)
            self.assertEqual(stats["stale_submission"], 1)
            self.assertEqual(stats["reuse_ignored"], 0)
            self.assertEqual(stats["outstanding"], 0)
            self.assertEqual(stats["by_priority"]["frontier_followup"]["expired"], 1)
            self.assertEqual(
                stats["by_priority"]["frontier_followup"]["stale_submission"],
                1,
            )

    def test_release_after_response_window_expires_without_decline_backoff(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler.time.time",
                return_value=float(lease["expires_at"]) + 1.0,
            ):
                service.release_supply_lease(
                    lease["lease_id"],
                    "gen0_peer0",
                    declined=True,
                )
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["expired"], 1)
            self.assertEqual(stats["declined"], 0)
            self.assertEqual(service._declined_supply_peers, {})

    def test_supply_transition_event_failure_keeps_or_releases_lease_atomically(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            with (
                patch.object(service, "_append_event", side_effect=OSError("disk full")),
                self.assertRaisesRegex(OSError, "disk full"),
            ):
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            self.assertEqual(service.status()["resource_supply"]["leases"], [])
            self.assertEqual(service.status()["resource_supply"]["stats"]["granted"], 0)

            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            lease = service.status()["resource_supply"]["leases"][0]
            with (
                patch.object(service, "_append_event", side_effect=OSError("disk full")),
                self.assertRaisesRegex(OSError, "disk full"),
            ):
                service.release_supply_lease(
                    lease["lease_id"],
                    "gen0_peer0",
                    declined=True,
                )
            self.assertEqual(len(service.status()["resource_supply"]["leases"]), 1)
            self.assertEqual(service.status()["resource_supply"]["stats"]["declined"], 0)
            service.release_supply_lease(lease["lease_id"], "gen0_peer0", declined=True)
            self.assertEqual(service.status()["resource_supply"]["stats"]["declined"], 1)

    def test_signal_and_terminal_audit_failure_release_unpublished_supply_claim(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            original_append = service._append_event
            original_release = allocator.release
            release_attempts = {"count": 0}

            def fail_supply_revoked(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_revoked":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            def fail_release_once(allocation_id: str) -> None:
                release_attempts["count"] += 1
                if release_attempts["count"] == 1:
                    raise OSError("registry unavailable")
                original_release(allocation_id)

            with (
                patch.object(service, "_append_event", side_effect=fail_supply_revoked),
                patch.object(
                    service,
                    "_write_supply_signal",
                    side_effect=OSError("signal unavailable"),
                ),
                patch.object(allocator, "release", side_effect=fail_release_once),
                self.assertRaisesRegex(OSError, "signal unavailable"),
            ):
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))

            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["granted"], 1)
            self.assertEqual(stats["revoked"], 0)
            self.assertEqual(stats["outstanding"], 1)
            self.assertTrue(service.status()["resource_supply"]["leases"][0]["release_pending"])
            self.assertIn(next(iter(allocator.claims)), service._pending_supply_releases)
            service._idle_supply_waiters.clear()
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["revoked"], 1)
            self.assertEqual(stats["outstanding"], 0)
            self.assertEqual(allocator.claims, {})

    def test_grant_audit_and_release_failure_remains_reconcilable(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            original_append = service._append_event

            def fail_grant(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_granted":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with (
                patch.object(service, "_append_event", side_effect=fail_grant),
                patch.object(allocator, "release", side_effect=OSError("registry unavailable")),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            supply = service.status()["resource_supply"]
            self.assertEqual(len(supply["leases"]), 1)
            self.assertTrue(supply["leases"][0]["release_pending"])
            lease_id = supply["leases"][0]["lease_id"]
            self.assertIn(lease_id, allocator.claims)

            service._idle_supply_waiters.clear()
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            self.assertEqual(service.status()["resource_supply"]["leases"], [])
            self.assertNotIn(lease_id, allocator.claims)
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["granted"], 0)
            self.assertEqual(stats["revoked"], 0)
            events = [
                json.loads(line)
                for line in (service.state_dir / "events.jsonl").read_text().splitlines()
            ]
            self.assertFalse(
                any(
                    event["event"] in {"supply_granted", "supply_revoked"}
                    and event.get("lease_id") == lease_id
                    for event in events
                )
            )

    def test_supply_audit_failure_cannot_duplicate_an_accepted_semantic_job(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            original_append = service._append_event

            def fail_supply_consumed(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_consumed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            request = {
                "command": [sys.executable, "-c", "pass"],
                "peer_id": "gen0_peer0",
                "generation_id": 0,
                "experiment_id": "one-semantic-job",
                "supply_lease_id": lease["lease_id"],
            }
            with patch.object(service, "_append_event", side_effect=fail_supply_consumed):
                first = service.submit(request)
                duplicate = service.submit(request)
                second = service.submit(
                    {
                        **request,
                        "experiment_id": "different-semantic-job",
                    }
                )
            self.assertEqual(duplicate.job_id, first.job_id)
            self.assertEqual(service._queue, [first.job_id, second.job_id])
            self.assertEqual(first.supply_claim_id, lease["lease_id"])
            self.assertEqual(second.supply_claim_id, "")
            self.assertEqual(
                set(service._semantic_jobs.values()),
                {first.job_id, second.job_id},
            )
            service.release_supply_lease(
                lease["lease_id"],
                "gen0_peer0",
                declined=True,
            )
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["consumed"], 1)
            self.assertEqual(stats["declined"], 0)
            self.assertEqual(stats["revoked"], 0)
            self.assertEqual(stats["reuse_ignored"], 1)

            resumed = ExperimentSchedulerService(
                run_dir=service.run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            resumed._recover_terminal_events()
            self.assertEqual(len(resumed._queue), 2)
            self.assertEqual(len(resumed._semantic_jobs), 2)

    def test_terminal_job_retains_pending_consumption_until_audit_recovers(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600, cohort_size=2)
            service.configure_generation_maturity(
                0,
                cohort_size=2,
                mature_target=2,
                count_callback=lambda: 0,
                identity_callback=set,
                distinct_peer_commitments=True,
            )
            service._refresh_mature_counts(HostSnapshot(8, 5, 5, 0, observed_at=0.5))
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            original_append = service._append_event

            def fail_consumed(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_consumed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            request = {
                "command": [sys.executable, "-c", "pass"],
                "peer_id": "gen0_peer0",
                "generation_id": 0,
                "experiment_id": "terminal-pending-consumption",
                "supply_lease_id": lease["lease_id"],
                "work_class": "mature",
            }
            with patch.object(service, "_append_event", side_effect=fail_consumed):
                job = service.submit(request)
                job.state = "completed"
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            self.assertEqual(job.supply_claim_id, lease["lease_id"])
            self.assertEqual(
                service.get_supply_lease("gen0_peer0", 0, lease["lease_id"]),
                {},
            )
            self.assertIn(lease["lease_id"], allocator.claims)
            maturity = service.status()["resource_supply"]["maturity"]["0"]
            self.assertEqual(maturity["priority_leases"], 0)
            self.assertEqual(maturity["needed_inflight"], 2)

            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=3.0))
            self.assertEqual(job.supply_claim_id, "")
            self.assertNotIn(lease["lease_id"], allocator.claims)
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["consumed"], 1)
            self.assertEqual(stats["revoked"], 0)

    def test_freeze_consumes_accepted_pending_supply_before_queue_rejection(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            original_append = service._append_event

            def fail_supply_consumed(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_consumed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with patch.object(service, "_append_event", side_effect=fail_supply_consumed):
                job = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "accepted-before-freeze",
                        "supply_lease_id": lease["lease_id"],
                    }
                )
            with patch.object(
                service,
                "_unlink_supply_signal",
                side_effect=OSError("unlink unavailable"),
            ):
                service.freeze_generation(0, "test boundary")
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["consumed"], 1)
            self.assertEqual(stats["revoked"], 0)
            self.assertEqual(service._jobs[job.job_id].state, "rejected")
            self.assertEqual(service._supply_leases, {})
            self.assertNotIn(lease["lease_id"], allocator.claims)

    def test_rejection_completes_when_supply_audit_and_signal_cleanup_both_fail(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            original_append = service._append_event

            def fail_supply_consumed(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_consumed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with patch.object(service, "_append_event", side_effect=fail_supply_consumed):
                job = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "rejected-after-double-cleanup-failure",
                        "supply_lease_id": lease["lease_id"],
                    }
                )
                with patch.object(
                    service,
                    "_unlink_supply_signal",
                    side_effect=OSError("unlink unavailable"),
                ):
                    service._queue.remove(job.job_id)
                    service._finish_without_launch(job, "test rejection")

            self.assertEqual(job.state, "rejected")
            self.assertEqual(job.supply_claim_id, "")
            self.assertEqual(service._supply_leases, {})
            self.assertNotIn(lease["lease_id"], allocator.claims)

    def test_unregister_cannot_revoke_supply_bound_to_an_accepted_job(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            original_append = service._append_event

            def fail_consumed_once(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_consumed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with patch.object(service, "_append_event", side_effect=fail_consumed_once):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "accepted-before-unregister",
                        "supply_lease_id": lease["lease_id"],
                    }
                )
            service.unregister_idle_supply("gen0_peer0", 0)
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["consumed"], 1)
            self.assertEqual(stats["revoked"], 0)

    def test_assessment_cannot_revoke_supply_bound_to_a_running_job(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            original_append = service._append_event

            def fail_consumed_once(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_consumed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with patch.object(service, "_append_event", side_effect=fail_consumed_once):
                job = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "accepted-before-assessment",
                        "supply_lease_id": lease["lease_id"],
                    }
                )
            service._queue.remove(job.job_id)
            job.state = "running"
            service.begin_assessment(0)
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["consumed"], 1)
            self.assertEqual(stats["revoked"], 0)
            self.assertEqual(job.supply_claim_id, "")

    def test_required_event_fsync_error_is_not_acknowledged_as_durable(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.os.fsync",
                    side_effect=OSError("uncertain fsync"),
                ),
                self.assertRaisesRegex(OSError, "uncertain fsync"),
            ):
                service.release_supply_lease(lease["lease_id"], "gen0_peer0", declined=True)
            self.assertEqual(service.status()["resource_supply"]["stats"]["declined"], 0)
            self.assertEqual(len(service.status()["resource_supply"]["leases"]), 1)
            events = [
                json.loads(line)
                for line in (service.state_dir / "events.jsonl").read_text().splitlines()
            ]
            self.assertFalse(any(event["event"] == "supply_declined" for event in events))
            service.release_supply_lease(lease["lease_id"], "gen0_peer0", declined=True)
            self.assertEqual(service.status()["resource_supply"]["stats"]["declined"], 1)

    def test_failed_submission_fsync_cannot_recover_as_a_ghost_job(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            request = {
                "command": [sys.executable, "-c", "pass"],
                "peer_id": "gen0_peer0",
                "generation_id": 0,
                "experiment_id": "single-semantic-retry",
            }
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.os.fsync",
                    side_effect=(OSError("uncertain fsync"), None),
                ),
                self.assertRaisesRegex(OSError, "uncertain fsync"),
            ):
                service.submit(request)
            self.assertEqual(service._jobs, {})
            self.assertFalse(
                any(
                    json.loads(line).get("event") == "submitted"
                    for line in (service.state_dir / "events.jsonl").read_text().splitlines()
                )
            )

            accepted = service.submit(request)
            resumed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            resumed._recover_terminal_events()
            self.assertEqual(resumed._queue, [accepted.job_id])
            self.assertEqual(list(resumed._semantic_jobs.values()), [accepted.job_id])

    def test_partial_event_tail_is_removed_before_the_next_submission(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            original_write = os.write
            write_attempts = {"count": 0}

            def partial_then_fail(fd: int, data: bytes | memoryview) -> int:
                write_attempts["count"] += 1
                if write_attempts["count"] == 1:
                    return original_write(fd, data[:12])
                raise OSError("write interrupted")

            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.os.write",
                    side_effect=partial_then_fail,
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.os.ftruncate",
                    side_effect=OSError("rollback unavailable"),
                ),
                self.assertRaisesRegex(OSError, "write interrupted"),
            ):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "partial-write",
                    }
                )

            accepted = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "accepted-after-partial-write",
                }
            )
            resumed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            resumed._recover_terminal_events()
            self.assertEqual(resumed._queue, [accepted.job_id])

    def test_unrecoverable_event_rollback_is_not_acknowledged(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.os.fsync",
                    side_effect=OSError("fsync unavailable"),
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.os.ftruncate",
                    side_effect=OSError("rollback unavailable"),
                ),
                self.assertRaisesRegex(OSError, "fsync unavailable"),
            ):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "uncertain-but-owned",
                    }
                )
            self.assertEqual(service._queue, [])
            self.assertEqual(service._jobs, {})

    def test_event_descriptor_close_failure_does_not_reject_a_durable_submission(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            with patch(
                "praxist.plugins.workflow_stages.research_loop.backend."
                "experiment_scheduler.os.close",
                side_effect=OSError("close unavailable"),
            ):
                accepted = service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "durable-before-close",
                    }
                )
            self.assertEqual(service._queue, [accepted.job_id])

    def test_retry_audit_failure_keeps_running_job_owned_until_retry_is_durable(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "retry-remains-owned",
                }
            )
            service._queue.remove(job.job_id)
            job.attempts = 1
            job.state = "running"
            original_append = service._append_event

            def fail_retry(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "retry_queued":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with (
                patch.object(service, "_append_event", side_effect=fail_retry),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service._retry_or_finish(job, 75)
            self.assertEqual(job.state, "running")
            self.assertEqual(service._queue, [])
            self.assertEqual(service._semantic_jobs[job.semantic_key], job.job_id)

    def test_launch_failure_retry_audit_failure_restores_queued_job(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "launch-failure-retry-audit",
                }
            )
            original_append = service._append_event

            def fail_retry(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "retry_queued":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with (
                patch.object(service, "_append_event", side_effect=fail_retry),
                patch.object(Path, "mkdir", side_effect=OSError("attempt directory unavailable")),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service._launch(job)
            self.assertEqual(job.state, "queued")
            self.assertEqual(service._queue, [job.job_id])
            self.assertNotIn(job.job_id, service._active)

    def test_post_start_launch_failure_retry_audit_failure_restores_queue(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "post-start-retry-audit",
                }
            )
            original_append = service._append_event

            def fail_retry(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "retry_queued":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with (
                patch.object(service, "_append_event", side_effect=fail_retry),
                patch.object(
                    service,
                    "_register_protected",
                    side_effect=OSError("manifest unavailable"),
                ),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service._launch(job)
            self.assertEqual(job.state, "queued")
            self.assertEqual(service._queue, [job.job_id])
            self.assertNotIn(job.job_id, service._active)

    def test_assessment_rechecks_ordinary_queue_after_rejection_audit_failure(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            service.open_generation(0, deadline=time.time() + 600)
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "ordinary-before-assessment",
                    "work_class": "ordinary",
                }
            )
            original_append = service._append_event

            def fail_completed(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "completed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with (
                patch.object(service, "_append_event", side_effect=fail_completed),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service.begin_assessment(0)
            self.assertEqual(job.state, "queued")
            self.assertEqual(service._queue, [job.job_id])
            with (
                patch.object(service, "_append_event", side_effect=fail_completed),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service._launch_ready_jobs()
            self.assertEqual(job.state, "queued")
            self.assertNotIn(job.job_id, service._active)

    def test_terminal_audit_failure_does_not_expose_undurable_terminal_state(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.infrastructure_retries = 0
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "notify-terminal-waiter",
                }
            )
            service._queue.remove(job.job_id)
            job.state = "running"
            result: dict[str, object] = {}
            waiter = threading.Thread(
                target=lambda: result.update(service.wait(job.job_id, 0.1)),
                daemon=True,
            )
            waiter.start()
            time.sleep(0.02)
            with (
                patch.object(service, "_append_event", side_effect=OSError("audit unavailable")),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service._retry_or_finish(job, 1)
            waiter.join(timeout=0.5)
            self.assertFalse(waiter.is_alive())
            self.assertTrue(result["timeout"])
            self.assertEqual(result["job"]["state"], "running")  # type: ignore[index]

    def test_queued_rejection_audit_failure_remains_queued_across_restart(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "cancel-remains-observable",
                }
            )
            result: dict[str, object] = {}
            waiter = threading.Thread(
                target=lambda: result.update(service.wait(job.job_id, 0.1)),
                daemon=True,
            )
            waiter.start()
            time.sleep(0.02)
            original_append = service._append_event

            def fail_completed(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "completed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with (
                patch.object(service, "_append_event", side_effect=fail_completed),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service.cancel_queued(job.job_id)
            waiter.join(timeout=0.5)
            self.assertFalse(waiter.is_alive())
            self.assertTrue(result["timeout"])
            self.assertEqual(job.state, "queued")
            self.assertEqual(result["job"]["state"], "queued")  # type: ignore[index]
            self.assertEqual(service._queue, [job.job_id])

            resumed = ExperimentSchedulerService(
                run_dir=service.run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            resumed._recover_terminal_events()
            self.assertEqual(resumed._queue, [job.job_id])
            self.assertEqual(resumed._jobs[job.job_id].state, "queued")

    def test_terminal_supply_release_failure_remains_visible_and_retries(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            original_release = allocator.release
            attempts = {"count": 0}

            def fail_once(allocation_id: str) -> None:
                attempts["count"] += 1
                if attempts["count"] == 1:
                    raise OSError("registry unavailable")
                original_release(allocation_id)

            with patch.object(allocator, "release", side_effect=fail_once):
                service.release_supply_lease(
                    lease["lease_id"],
                    "gen0_peer0",
                    declined=True,
                )
                first = service.status()["resource_supply"]
                self.assertEqual(first["stats"]["declined"], 0)
                self.assertEqual(first["stats"]["outstanding"], 1)
                self.assertTrue(first["leases"][0]["release_pending"])
                self.assertIn(lease["lease_id"], allocator.claims)
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))

            final = service.status()["resource_supply"]
            self.assertEqual(final["stats"]["declined"], 1)
            self.assertEqual(final["stats"]["outstanding"], 0)
            self.assertNotIn(lease["lease_id"], allocator.claims)

    def test_rejected_job_supply_claim_release_retries_during_reconcile(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "terminal-claim-retry",
                    "supply_lease_id": lease["lease_id"],
                }
            )
            with patch.object(allocator, "release", side_effect=OSError("registry unavailable")):
                service._queue.remove(job.job_id)
                service._finish_without_launch(job, "task_contract_rejected")
            self.assertEqual(job.state, "rejected")
            self.assertTrue(job.public()["supply_release_pending"])
            self.assertIn(lease["lease_id"], allocator.claims)
            with self.assertRaisesRegex(ExperimentRejected, "resource release is still pending"):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "terminal-claim-retry",
                        "retry_terminal": True,
                    }
                )

            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            self.assertFalse(job.public()["supply_release_pending"])
            self.assertNotIn(lease["lease_id"], allocator.claims)
            retried = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "terminal-claim-retry",
                    "retry_terminal": True,
                }
            )
            self.assertEqual(retried.job_id, job.job_id)
            self.assertEqual(retried.state, "queued")

    def test_signal_unlink_failure_cannot_strand_terminal_supply_claim(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            with patch.object(
                service,
                "_unlink_supply_signal",
                side_effect=OSError("unlink unavailable"),
            ):
                service.release_supply_lease(
                    lease["lease_id"],
                    "gen0_peer0",
                    declined=True,
                )
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["declined"], 1)
            self.assertEqual(stats["outstanding"], 0)
            self.assertNotIn(lease["lease_id"], allocator.claims)
            self.assertIn(("gen0_peer0", 0, "frontier_followup"), service._declined_supply_peers)

    def test_stop_continues_when_supply_revocation_audit_is_unavailable(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",), ("cpu",)))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=2),
                allocator=allocator,
            )
            service._acquire_owner_lock()
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer1",
                    "generation_id": 0,
                    "experiment_id": "queued-before-stop",
                }
            )
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            original_append = service._append_event

            def fail_supply_revoke(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_revoked":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with (
                patch.object(service, "_append_event", side_effect=fail_supply_revoke),
                patch.object(
                    service,
                    "_unlink_supply_signal",
                    side_effect=OSError("unlink unavailable"),
                ),
            ):
                service.stop()
                service._launch_ready_jobs()
            self.assertEqual(service._queue, [])
            self.assertEqual(service._supply_leases, {})
            self.assertEqual(service.status()["running"], 0)
            self.assertEqual(service._jobs[job.job_id].state, "rejected")
            self.assertEqual(allocator.claims, {})

    def test_stop_closes_transport_even_when_queued_terminal_audit_fails(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_BlockedAllocator(""),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "stop-audit-failure",
                }
            )
            service.start()
            original_append = service._append_event

            def fail_completed(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "completed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with (
                patch.object(service, "_append_event", side_effect=fail_completed),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service.stop()
            self.assertFalse(service.endpoint.exists())
            self.assertFalse((service.state_dir / "endpoint.json").exists())
            self.assertNotIn("PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT", os.environ)
            self.assertEqual(job.state, "queued")

            resumed = ExperimentSchedulerService(
                run_dir=service.run_dir,
                settings=_settings(maximum=1),
                allocator=_BlockedAllocator(""),
            )
            resumed._recover_terminal_events()
            self.assertEqual(resumed._queue, [])
            self.assertEqual(resumed._jobs[job.job_id].state, "rejected")
            resumed.open_generation(0, deadline=time.time() + 600)
            replacement = resumed.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "replacement-after-explicit-resume",
                }
            )
            self.assertEqual(resumed._jobs[job.job_id].state, "rejected")
            self.assertEqual(resumed._queue, [replacement.job_id])

    def test_stop_fence_audit_failure_retains_owner_until_retry_succeeds(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_BlockedAllocator(""),
            )
            service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "queued-before-fence-failure",
                }
            )
            service.start()
            original_append = service._append_event

            failures = {"remaining": 2}

            def fail_stop_fence(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "admission_closed" and failures["remaining"]:
                    failures["remaining"] -= 1
                    raise OSError("fence unavailable")
                original_append(payload, required=required)

            with patch.object(service, "_append_event", side_effect=fail_stop_fence):
                with self.assertRaisesRegex(OSError, "fence unavailable"):
                    service.stop()
                deadline = time.time() + 3
                while service._owner_lock_fd is not None and time.time() < deadline:
                    time.sleep(0.05)

            self.assertFalse(service.endpoint.exists())
            self.assertFalse((service.state_dir / "endpoint.json").exists())
            self.assertNotIn("PRAXIST_EXPERIMENT_SCHEDULER_ENDPOINT", os.environ)
            self.assertIsNone(service._owner_lock_fd)
            service._worker_thread.join(timeout=3)
            self.assertFalse(service._worker_thread.is_alive())
            events = [
                json.loads(line)
                for line in (service.state_dir / "events.jsonl").read_text().splitlines()
            ]
            self.assertTrue(any(event["event"] == "admission_closed" for event in events))

            successor = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_BlockedAllocator(""),
            )
            successor.start()
            successor.stop()

    def test_stop_retries_pending_host_claim_before_releasing_owner(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.start()
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease_id = service.status()["resource_supply"]["leases"][0]["lease_id"]
            original_release = allocator.release
            release_calls = {"count": 0}

            def fail_release_once(allocation_id: str) -> None:
                release_calls["count"] += 1
                if release_calls["count"] == 1:
                    raise OSError("registry unavailable")
                original_release(allocation_id)

            clear_calls = {"count": 0}

            def fail_clear_once(owner_id: str) -> None:
                clear_calls["count"] += 1
                if clear_calls["count"] == 1:
                    raise OSError("registry unavailable")
                allocator.claims.clear()

            with (
                patch.object(allocator, "release", side_effect=fail_release_once),
                patch.object(
                    allocator,
                    "clear_supply",
                    side_effect=fail_clear_once,
                    create=True,
                ),
            ):
                service.stop()

            self.assertGreaterEqual(release_calls["count"], 2)
            self.assertNotIn(lease_id, allocator.claims)
            self.assertIsNone(service._owner_lock_fd)

    def test_stop_releases_queued_job_claim_after_terminal_audit_retry(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            allocator.active.add("block-launch")
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service.start()
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            lease = service.status()["resource_supply"]["leases"][0]
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "queued-with-consumed-claim",
                    "supply_lease_id": lease["lease_id"],
                }
            )
            self.assertEqual(job.supply_claim_id, lease["lease_id"])
            original_append = service._append_event
            completed_failures = {"remaining": 1}

            def fail_completed_once(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "completed" and completed_failures["remaining"]:
                    completed_failures["remaining"] -= 1
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            clear_failures = {"remaining": 1}

            def fail_clear_once(owner_id: str) -> None:
                del owner_id
                if clear_failures["remaining"]:
                    clear_failures["remaining"] -= 1
                    raise OSError("registry unavailable")
                allocator.claims.clear()

            with (
                patch.object(service, "_append_event", side_effect=fail_completed_once),
                patch.object(
                    allocator,
                    "clear_supply",
                    side_effect=fail_clear_once,
                    create=True,
                ),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service.stop()

            deadline = time.time() + 3
            while service._owner_lock_fd is not None and time.time() < deadline:
                time.sleep(0.05)
            self.assertNotIn(lease["lease_id"], allocator.claims)
            self.assertEqual(job.supply_claim_id, "")
            self.assertIsNone(service._owner_lock_fd)

    def test_rpc_returns_structured_error_for_malformed_request(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_BlockedAllocator(""),
            )
            service.start()
            try:
                with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
                    client.connect(str(service.endpoint))
                    client.sendall(b"{not-json}\n")
                    response = json.loads(client.makefile("rb").readline())
                self.assertFalse(response["ok"])
                self.assertIn("JSONDecodeError", response["error"])
            finally:
                service.stop()

    def test_supply_lease_expiry_release_and_pending_submission_edges(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",), ("cpu",), ("cpu",)))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=3),
                allocator=allocator,
            )
            service.open_generation(0, deadline=time.time() + 3600)
            now = time.time()

            expired_reregister = _SupplyLease(
                "expired-reregister",
                "gen0_peer0",
                0,
                ("cpu",),
                "frontier_followup",
                now - 20,
                now - 10,
            )
            service._supply_leases[expired_reregister.lease_id] = expired_reregister
            allocator.claims[expired_reregister.lease_id] = ("cpu",)
            self.assertEqual(service.register_idle_supply("gen0_peer0", 0), {})
            self.assertNotIn(expired_reregister.lease_id, allocator.claims)

            expired_fetch = _SupplyLease(
                "expired-fetch",
                "gen0_peer1",
                0,
                ("cpu",),
                "frontier_followup",
                now - 20,
                now - 10,
            )
            service._supply_leases[expired_fetch.lease_id] = expired_fetch
            allocator.claims[expired_fetch.lease_id] = ("cpu",)
            self.assertEqual(service.get_supply_lease("gen0_peer1", 0, expired_fetch.lease_id), {})
            self.assertNotIn(expired_fetch.lease_id, allocator.claims)

            released = _SupplyLease(
                "ordinary-release",
                "gen0_peer2",
                0,
                ("cpu",),
                "frontier_followup",
                now,
                now + 600,
            )
            service._supply_leases[released.lease_id] = released
            allocator.claims[released.lease_id] = ("cpu",)
            service.release_supply_lease(released.lease_id, "gen0_peer2")
            self.assertNotIn(released.lease_id, allocator.claims)

            service._pending_supply_releases["pending-release"] = (
                "revoked",
                "cleanup_pending",
                None,
                True,
            )
            submitted = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer3",
                    "generation_id": 0,
                    "experiment_id": "stale-pending-lease",
                    "supply_lease_id": "pending-release",
                }
            )
            self.assertEqual(submitted.supply_claim_id, "")
            self.assertEqual(service.status()["resource_supply"]["stats"]["stale_submission"], 1)

    def test_stop_compacts_all_pending_supply_release_shapes(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",), ("cpu",), ("cpu",)))
            allocator.clear_supply = MagicMock(side_effect=lambda _owner: allocator.claims.clear())
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=3),
                allocator=allocator,
            )
            now = time.time()
            leases = {
                lease_id: _SupplyLease(
                    lease_id,
                    f"gen0_peer{index}",
                    0,
                    ("cpu",),
                    "frontier_followup",
                    now,
                    now + 600,
                )
                for index, lease_id in enumerate(("grant-failed", "audit-pending", "recorded"))
            }
            service._supply_leases.update(leases)
            allocator.claims.update({lease_id: ("cpu",) for lease_id in leases})
            service._pending_supply_releases.update(
                {
                    "missing": ("revoked", "cleanup", None, True),
                    "grant-failed": ("revoked", "grant_audit_failed", None, False),
                    "audit-pending": (
                        "revoked",
                        "cleanup",
                        {"cleanup_source": "test"},
                        False,
                    ),
                    "recorded": ("revoked", "cleanup", None, True),
                }
            )
            service._acquire_owner_lock()
            with patch.object(service, "_revoke_supply_locked"):
                service.stop()
            self.assertEqual(service._pending_supply_releases, {})
            self.assertEqual(service._supply_leases, {})
            self.assertEqual(allocator.claims, {})
            events = [
                json.loads(line)
                for line in (service.state_dir / "events.jsonl").read_text().splitlines()
            ]
            audit_event = next(
                event
                for event in events
                if event.get("event") == "supply_revoked"
                and event.get("lease_id") == "audit-pending"
            )
            self.assertEqual(audit_event["cleanup_source"], "test")

    def test_stop_clears_host_claim_when_supply_terminal_cleanup_is_unavailable(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _SupplyAllocator((("cpu",),))
            allocator.clear_supply = MagicMock(side_effect=lambda _owner: allocator.claims.clear())
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service._acquire_owner_lock()
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            original_append = service._append_event

            def fail_revoked(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_revoked":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with (
                patch.object(service, "_append_event", side_effect=fail_revoked),
                patch.object(
                    service,
                    "_unlink_supply_signal",
                    side_effect=OSError("unlink unavailable"),
                ),
                patch.object(allocator, "release", side_effect=OSError("registry unavailable")),
            ):
                service.stop()

            self.assertEqual(allocator.claims, {})
            allocator.clear_supply.assert_called_once_with(service.supply_owner_id)
            pending = service.status()["resource_supply"]["leases"]
            self.assertEqual(len(pending), 1)
            self.assertTrue(pending[0]["release_pending"])

    def test_supply_ledger_recovers_stats_tombstone_and_dangling_grant(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            first = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            events = [
                {
                    "event": "supply_granted",
                    "lease_id": "consumed-lease",
                    "priority": "mature",
                },
                {
                    "event": "supply_consumed",
                    "lease_id": "consumed-lease",
                    "priority": "mature",
                },
                {
                    "event": "supply_reuse_ignored",
                    "lease_id": "consumed-lease",
                },
                {
                    "event": "supply_granted",
                    "lease_id": "dangling-lease",
                    "priority": "frontier_followup",
                },
                {
                    "event": "supply_granted",
                    "lease_id": "accepted-before-terminal",
                    "priority": "mature",
                    "recorded_at": 40.0,
                },
                {
                    "event": "submitted",
                    "recorded_at": 90.0,
                    "environment_values": {
                        "PRAXIST_RESOURCE_SUPPLY_LEASE_ID": "accepted-before-terminal"
                    },
                },
            ]
            (first.state_dir / "events.jsonl").write_text(
                "".join(json.dumps(event) + "\n" for event in events),
                encoding="utf-8",
            )
            first._recover_terminal_events()
            stats = first.status()["resource_supply"]["stats"]
            self.assertEqual(stats["granted"], 3)
            self.assertEqual(stats["consumed"], 2)
            self.assertEqual(stats["revoked"], 1)
            self.assertEqual(stats["reuse_ignored"], 1)
            self.assertIn("consumed-lease", first._consumed_supply_leases)
            self.assertIn("accepted-before-terminal", first._consumed_supply_leases)
            first.open_generation(1, deadline=time.time() + 600)
            self.assertIn("consumed-lease", first._consumed_supply_leases)

            resumed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            resumed._recover_terminal_events()
            self.assertEqual(resumed.status()["resource_supply"]["stats"], stats)

    def test_supply_ledger_recovers_explicit_terminal_retry_acceptance(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            now = time.time()
            service._recover_supply_events(
                [
                    {
                        "event": "supply_granted",
                        "lease_id": "retry-accepted-lease",
                        "priority": "ordinary",
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "admissible_profiles": ["cpu"],
                        "issued_at": now - 10,
                        "expires_at": now + 60,
                    },
                    {
                        "event": "retry_queued",
                        "retry_reason": "explicit_terminal_retry",
                        "supply_lease_id": "retry-accepted-lease",
                        "supply_lease_eligible": True,
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "profile": "cpu",
                        "work_class": "ordinary",
                        "recorded_at": now,
                    },
                ]
            )

            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["granted"], 1)
            self.assertEqual(stats["consumed"], 1)
            self.assertEqual(stats["revoked"], 0)
            self.assertIn("retry-accepted-lease", service._consumed_supply_leases)

    def test_restart_does_not_consume_an_ineligible_mature_supply_submission(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600, cohort_size=1)
            service.configure_generation_maturity(
                0,
                cohort_size=1,
                mature_target=1,
                count_callback=lambda: 0,
            )
            service.register_idle_supply("gen0_peer0", 0)
            snapshot = HostSnapshot(8, 5, 5, 0, observed_at=1.0)
            service._refresh_mature_counts(snapshot)
            service._reconcile_supply(snapshot)
            lease = service.status()["resource_supply"]["leases"][0]
            self.assertEqual(lease["priority"], "mature")
            original_append = service._append_event

            def fail_supply_revoked(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_revoked":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with patch.object(service, "_append_event", side_effect=fail_supply_revoked):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "ordinary-work-on-mature-supply",
                        "work_class": "ordinary",
                        "supply_lease_id": lease["lease_id"],
                        "environment": {"PRAXIST_RESOURCE_SUPPLY_LEASE_ID": lease["lease_id"]},
                    }
                )
            pending = service.status()["resource_supply"]["leases"]
            self.assertEqual(len(pending), 1)
            self.assertTrue(pending[0]["release_pending"])
            self.assertEqual(
                service.get_supply_lease("gen0_peer0", 0, lease["lease_id"]),
                {},
            )

            resumed = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            resumed._recover_terminal_events()
            stats = resumed.status()["resource_supply"]["stats"]
            self.assertEqual(stats["consumed"], 0)
            self.assertEqual(stats["revoked"], 1)
            self.assertEqual(stats["conversion_rate"], 0.0)

    def test_pending_consumption_counts_mature_job_and_lease_only_once(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=3),
                allocator=_SupplyAllocator((("cpu",), ("cpu",), ("cpu",))),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600, cohort_size=3)
            service.configure_generation_maturity(
                0,
                cohort_size=3,
                mature_target=3,
                count_callback=lambda: 0,
            )
            service.register_idle_supply("gen0_peer0", 0)
            snapshot = HostSnapshot(8, 5, 5, 0, observed_at=1.0)
            service._refresh_mature_counts(snapshot)
            service._reconcile_supply(snapshot)
            lease = service.status()["resource_supply"]["leases"][0]
            original_append = service._append_event

            def fail_supply_consumed(payload: dict[str, object], *, required: bool = False) -> None:
                if payload.get("event") == "supply_consumed":
                    raise OSError("audit unavailable")
                original_append(payload, required=required)

            with patch.object(service, "_append_event", side_effect=fail_supply_consumed):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "mature-pending-consumption",
                        "work_class": "mature",
                        "supply_lease_id": lease["lease_id"],
                    }
                )
            maturity = service.status()["resource_supply"]["maturity"]["0"]
            self.assertEqual(maturity["inflight_jobs"], 1)
            self.assertEqual(maturity["priority_leases"], 0)
            self.assertEqual(maturity["needed_inflight"], 2)

    def test_reused_consumed_lease_keeps_its_priority_accounting(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600, cohort_size=1)
            service.configure_generation_maturity(
                0,
                cohort_size=1,
                mature_target=1,
                count_callback=lambda: 0,
            )
            service.register_idle_supply("gen0_peer0", 0)
            snapshot = HostSnapshot(8, 5, 5, 0, observed_at=1.0)
            service._refresh_mature_counts(snapshot)
            service._reconcile_supply(snapshot)
            lease = service.status()["resource_supply"]["leases"][0]
            service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "consume-mature-lease",
                    "work_class": "mature",
                    "supply_lease_id": lease["lease_id"],
                }
            )
            service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "reuse-mature-lease",
                    "work_class": "mature",
                    "supply_lease_id": lease["lease_id"],
                }
            )
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["reuse_ignored"], 1)
            self.assertEqual(stats["by_priority"]["mature"]["reuse_ignored"], 1)

    def test_restart_classifies_elapsed_unanswered_supply_as_expired(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service._append_event(
                {
                    "event": "supply_granted",
                    "lease_id": "elapsed-before-restart",
                    "priority": "mature",
                    "expires_at": time.time() - 400,
                },
                required=True,
            )
            service._recover_terminal_events()
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["expired"], 1)
            self.assertEqual(stats["revoked"], 0)

    def test_live_process_owns_semantic_identity_over_older_terminal_job(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            common = {
                "generation_id": 0,
                "peer_id": "gen0_peer0",
                "experiment_id": "same-semantic-experiment",
                "profile": "cpu",
                "work_class": "ordinary",
                "command": [sys.executable, "-c", "pass"],
                "environment_values": {},
                "attempt": 0,
            }
            events = [
                {"event": "submitted", "job_id": "old-job", **common},
                {
                    "event": "completed",
                    "job_id": "old-job",
                    **common,
                    "state": "completed",
                    "exit_code": 0,
                    "recorded_at": 1.0,
                },
                {"event": "submitted", "job_id": "live-job", **common},
                {
                    "event": "launched",
                    "job_id": "live-job",
                    **common,
                    "pid": 123,
                    "pgid": 123,
                    "allocation_id": "live-allocation",
                    "recorded_at": 2.0,
                },
            ]
            (service.state_dir / "events.jsonl").write_text(
                "".join(json.dumps(event) + "\n" for event in events),
                encoding="utf-8",
            )
            with (
                patch.object(service, "_event_process_matches", return_value=True),
                patch.object(service, "_register_protected"),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.process_group_alive",
                    return_value=True,
                ),
            ):
                service._recover_terminal_events()
            semantic = next(iter(service._semantic_jobs))
            self.assertEqual(service._semantic_jobs[semantic], "live-job")
            self.assertIn("live-job", service._active)
            self.assertNotIn("old-job", service._jobs)

    def test_semantic_duplicate_does_not_consume_supply_lease(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            settings = _settings(maximum=2)
            settings.supply_idle_samples = 2
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=_SupplyAllocator((("cpu",), ("cpu",))),
            )
            original = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "existing-work",
                }
            )
            service.open_generation(0, deadline=time.time() + 600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            lease = service.status()["resource_supply"]["leases"][0]
            duplicate = service.submit(
                {
                    "command": [sys.executable, "-c", "raise SystemExit(99)"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "existing-work",
                    "supply_lease_id": lease["lease_id"],
                }
            )
            self.assertEqual(duplicate.job_id, original.job_id)
            self.assertEqual(len(service.status()["resource_supply"]["leases"]), 1)

    def test_closing_revokes_supply_without_disturbing_running_jobs(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 2
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.open_generation(4, deadline=time.time() + 600)
            service.register_idle_supply("gen4_peer0", 4)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            self.assertEqual(len(service.status()["resource_supply"]["leases"]), 1)
            service.freeze_generation(4, "generation_closing")
            supply = service.status()["resource_supply"]
            self.assertEqual(supply["leases"], [])
            self.assertEqual(supply["idle_waiters"], 0)
            self.assertFalse((run_dir / "gen_4" / "resource_supply" / "gen4_peer0.json").exists())

    def test_invalid_host_claim_withdraws_published_supply_lease(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 2
            allocator = _SupplyAllocator((("cpu",),))
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=allocator,
            )
            service.open_generation(1, deadline=time.time() + 600)
            service.register_idle_supply("gen1_peer0", 1)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            lease = service.status()["resource_supply"]["leases"][0]
            path = run_dir / "gen_1" / "resource_supply" / "gen1_peer0.json"
            allocator.claims.pop(lease["lease_id"])
            service._reconcile_supply(HostSnapshot(8, 95, 5, 0, observed_at=3.0))
            supply = service.status()["resource_supply"]
            self.assertEqual(supply["leases"], [])
            self.assertEqual(supply["stats"]["revoked"], 1)
            self.assertFalse(path.exists())

    def test_pressure_resample_keeps_supply_lease_until_submission(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 2
            allocator = ResourceAllocator(
                settings,
                observer=_StaticObserver(),
                registry=HostAllocationRegistry(Path(td) / "registry"),
            )
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=allocator,
            )
            service.open_generation(1, deadline=time.time() + 600)
            service.register_idle_supply("gen1_peer0", 1)
            low = HostSnapshot(8, 5, 5, 0, observed_at=1.0)
            service._reconcile_supply(low)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            lease = service.status()["resource_supply"]["leases"][0]

            allocator._last_snapshot = HostSnapshot(8, 99, 5, 0, observed_at=3.0)
            service._reconcile_supply(allocator.snapshot)
            self.assertEqual(
                service.get_supply_lease("gen1_peer0", 1, lease["lease_id"])["lease_id"],
                lease["lease_id"],
            )

            service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "experiment_id": "planned-followup",
                    "generation_id": 1,
                    "peer_id": "gen1_peer0",
                    "profile": "cpu",
                    "supply_lease_id": lease["lease_id"],
                }
            )
            supply = service.status()["resource_supply"]
            self.assertEqual(supply["stats"]["consumed"], 1)
            self.assertEqual(supply["leases"], [])

    def test_unproductive_peer_unregisters_waiter_and_published_lease(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 2
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.open_generation(2, deadline=time.time() + 600)
            service.register_idle_supply("gen2_peer0", 2)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            lease_id = service.status()["resource_supply"]["leases"][0]["lease_id"]
            service.unregister_idle_supply("gen2_peer0", 2)
            supply = service.status()["resource_supply"]
            self.assertEqual(supply["leases"], [])
            self.assertEqual(supply["stats"]["consumed"], 0)
            self.assertEqual(supply["stats"]["revoked"], 1)
            self.assertNotIn(lease_id, service._consumed_supply_leases)
            service.open_generation(3, deadline=time.time() + 600)
            self.assertEqual(service._consumed_supply_leases, {})

    def test_declined_supply_can_later_receive_changed_mature_priority(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 2
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=settings,
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.open_generation(0, deadline=time.time() + 600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            lease_id = service.status()["resource_supply"]["leases"][0]["lease_id"]
            service.release_supply_lease(lease_id, "gen0_peer0", declined=True)
            self.assertEqual(service.status()["resource_supply"]["stats"]["declined"], 1)
            service.register_idle_supply("gen0_peer0", 0)
            self.assertEqual(service.status()["resource_supply"]["idle_waiters"], 1)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=3.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=4.0))
            self.assertEqual(service.status()["resource_supply"]["leases"], [])

            service._assessment_generations.add(0)
            service._generation_mature_targets[0] = 1
            service._generation_cohort_sizes[0] = 1
            service._mature_completed[0] = 0
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=5.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=6.0))
            leases = service.status()["resource_supply"]["leases"]
            self.assertEqual(len(leases), 1)
            self.assertEqual(leases[0]["priority"], "mature")
            service.open_generation(1, deadline=time.time() + 600)
            service.register_idle_supply("gen1_peer0", 1)
            self.assertEqual(service.status()["resource_supply"]["idle_waiters"], 1)

    def test_repeated_same_priority_declines_back_off_and_submission_resets_them(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.settings.supply_idle_samples = 1
            service.open_generation(0, deadline=time.time() + 3600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            first = service.status()["resource_supply"]["leases"][0]
            service.release_supply_lease(first["lease_id"], "gen0_peer0", declined=True)
            key = ("gen0_peer0", 0, "frontier_followup")
            first_expiry, first_count = service._declined_supply_peers[key]
            self.assertEqual(first_count, 1)

            service._declined_supply_peers[key] = (0.0, first_count)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            second = service.status()["resource_supply"]["leases"][0]
            service.release_supply_lease(second["lease_id"], "gen0_peer0", declined=True)
            second_expiry, second_count = service._declined_supply_peers[key]
            self.assertEqual(second_count, 2)
            self.assertGreater(second_expiry - time.time(), first_expiry - time.time())

            service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "newly-justified-plan",
                }
            )
            self.assertNotIn(key, service._declined_supply_peers)
            stats = service.status()["resource_supply"]["stats"]
            self.assertEqual(stats["declined"], 2)
            self.assertEqual(stats["outstanding"], 0)
            events = [
                json.loads(line)
                for line in (service.state_dir / "events.jsonl").read_text().splitlines()
            ]
            declines = [event for event in events if event["event"] == "supply_declined"]
            self.assertEqual([event["cooldown_seconds"] for event in declines], [60.0, 120.0])

    def test_failed_submission_does_not_reset_decline_backoff(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            key = ("gen0_peer0", 0, "frontier_followup")
            expected = (time.time() + 300, 3)
            service._declined_supply_peers[key] = expected
            with (
                patch.object(service, "_append_event", side_effect=OSError("audit unavailable")),
                self.assertRaisesRegex(OSError, "audit unavailable"),
            ):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "submission-not-durable",
                    }
                )
            self.assertEqual(service._declined_supply_peers[key], expected)

    def test_scheduler_start_removes_stale_supply_files(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            stale = run_dir / "gen_2" / "resource_supply" / "gen2_peer1.json"
            stale.parent.mkdir(parents=True)
            stale.write_text("{}")
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_GPUAllocator(""),
            )
            service.start()
            try:
                self.assertFalse(stale.exists())
            finally:
                service.stop()

    def test_scheduler_stop_materializes_empty_supply_status(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 2
            service = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=settings,
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service._acquire_owner_lock()
            service.open_generation(0, deadline=time.time() + 600)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0, observed_at=2.0))
            service._write_snapshot(force=True)
            self.assertEqual(
                len(
                    json.loads((service.state_dir / "status.json").read_text())["resource_supply"][
                        "leases"
                    ]
                ),
                1,
            )
            service.stop()
            final_status = json.loads((service.state_dir / "status.json").read_text())
            self.assertEqual(final_status["resource_supply"]["leases"], [])
            self.assertEqual(final_status["resource_supply"]["idle_waiters"], 0)

    def test_mature_debt_issues_three_bounded_priority_leases_per_missing_result(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            settings = _settings(maximum=12)
            settings.supply_idle_samples = 2
            allocator = _SupplyAllocator(tuple(("cpu",) for _ in range(12)))
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=settings,
                allocator=allocator,
            )
            service.open_generation(0, deadline=time.time() + 600, cohort_size=12)
            completed = {"count": 0}
            service.configure_generation_maturity(
                0,
                cohort_size=12,
                mature_target=3,
                count_callback=lambda: completed["count"],
            )
            for index in range(12):
                service.register_idle_supply(f"gen0_peer{index}", 0)
            first = HostSnapshot(16, 5, 5, 0, observed_at=1.0)
            second = HostSnapshot(16, 5, 5, 0, observed_at=2.0)
            service._refresh_mature_counts(first)
            service._reconcile_supply(first)
            service._refresh_mature_counts(second)
            service._reconcile_supply(second)
            supply = service.status()["resource_supply"]
            self.assertEqual(len(supply["leases"]), 12)
            self.assertEqual(
                sum(lease["priority"] == "mature" for lease in supply["leases"]),
                9,
            )
            self.assertEqual(
                sum(lease["priority"] == "frontier_followup" for lease in supply["leases"]),
                3,
            )
            self.assertEqual(supply["maturity"]["0"]["debt"], 3)
            self.assertEqual(supply["maturity"]["0"]["target_inflight"], 9)

            completed["count"] = 3
            third = HostSnapshot(16, 5, 5, 0, observed_at=3.0)
            service._refresh_mature_counts(third)
            service._reconcile_supply(third)
            remaining = service.status()["resource_supply"]["leases"]
            self.assertEqual(len(remaining), 3)
            self.assertEqual({lease["priority"] for lease in remaining}, {"frontier_followup"})

    def test_first_wave_preserves_exploration_and_assigns_direct_mature_work(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=12),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.open_generation(4, deadline=time.time() + 600, cohort_size=12)
            service.configure_generation_maturity(
                4,
                cohort_size=12,
                mature_target=3,
                count_callback=lambda: 0,
            )
            self.assertEqual(
                service.generation_advice("gen4_peer0", 4)["first_wave"], "direct_mature"
            )
            self.assertEqual(
                service.generation_advice("gen4_peer2", 4)["first_wave"], "direct_mature"
            )
            self.assertEqual(service.generation_advice("gen4_peer3", 4)["first_wave"], "explore")

    def test_first_wave_ranks_surviving_sparse_peer_ids(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=3),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.open_generation(
                4,
                deadline=time.time() + 600,
                cohort_size=3,
                peer_ids=("gen4_peer1", "gen4_peer2", "gen4_peer3"),
            )
            service.configure_generation_maturity(
                4,
                cohort_size=3,
                mature_target=1,
                count_callback=lambda: 0,
            )
            self.assertEqual(
                service.generation_advice("gen4_peer1", 4)["first_wave"], "direct_mature"
            )
            self.assertEqual(service.generation_advice("gen4_peer2", 4)["first_wave"], "explore")

    def test_stop_without_owner_cannot_mutate_live_run_ledger(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            owner = ExperimentSchedulerService(
                run_dir=run_dir,
                settings=_settings(maximum=1),
                allocator=_BlockedAllocator(""),
            )
            owner.start()
            try:
                events_path = owner.state_dir / "events.jsonl"
                events_before = events_path.read_bytes() if events_path.exists() else b""
                non_owner = ExperimentSchedulerService(
                    run_dir=run_dir,
                    settings=_settings(maximum=1),
                    allocator=_BlockedAllocator(""),
                )
                non_owner.stop()
                self.assertEqual(
                    events_path.read_bytes() if events_path.exists() else b"",
                    events_before,
                )
                self.assertFalse(owner._admission_closed)
            finally:
                owner.stop()

    def test_open_generation_retires_previous_maturity_callback(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=2)
            )
            service.open_generation(0, deadline=time.time() + 600, cohort_size=4)

            def callback() -> int:
                return 0

            service.configure_generation_maturity(
                0,
                cohort_size=4,
                mature_target=1,
                count_callback=callback,
            )
            service.open_generation(1, deadline=time.time() + 600, cohort_size=4)
            self.assertEqual(service._mature_count_callbacks, {})
            self.assertNotIn(0, service._generation_mature_targets)
            self.assertNotIn(0, service._mature_completed)

    def test_hard_quorum_commitments_are_distinct_uncompleted_peers(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=10)
            )
            service.open_generation(0, deadline=time.time() + 600, cohort_size=4)
            completed = {"gen0_peer0"}
            service.configure_generation_maturity(
                0,
                cohort_size=4,
                mature_target=2,
                count_callback=lambda: len(completed),
                identity_callback=lambda: set(completed),
                distinct_peer_commitments=True,
            )
            for peer_id in ("gen0_peer0", "gen0_peer1"):
                for index in range(3):
                    service.submit(
                        {
                            "command": [sys.executable, "-c", "pass"],
                            "peer_id": peer_id,
                            "generation_id": 0,
                            "experiment_id": f"{peer_id}-{index}",
                            "work_class": "mature",
                        }
                    )
            service._refresh_mature_counts(HostSnapshot(8, 5, 5, 0, observed_at=1.0))
            maturity = service.status()["resource_supply"]["maturity"]["0"]
            self.assertEqual(maturity["completed"], 1)
            self.assertEqual(maturity["inflight_jobs"], 1)
            self.assertEqual(maturity["target_inflight"], 3)
            self.assertEqual(maturity["needed_inflight"], 2)

    def test_completed_mature_peer_can_receive_frontier_followup_supply(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 1
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=settings,
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.open_generation(0, deadline=time.time() + 600, cohort_size=1)
            service.configure_generation_maturity(
                0,
                cohort_size=1,
                mature_target=1,
                count_callback=lambda: 1,
                identity_callback=lambda: {"gen0_peer0"},
                distinct_peer_commitments=True,
            )
            snapshot = HostSnapshot(8, 5, 5, 0, observed_at=1.0)
            service._refresh_mature_counts(snapshot)
            service.register_idle_supply("gen0_peer0", 0)
            service._reconcile_supply(snapshot)
            leases = service.status()["resource_supply"]["leases"]
            self.assertEqual(len(leases), 1)
            self.assertEqual(leases[0]["priority"], "frontier_followup")
            self.assertEqual(service.status()["resource_supply"]["idle_waiters"], 0)

    def test_completed_mature_waiter_survives_debt_then_receives_followup(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            settings = _settings(maximum=2)
            settings.supply_idle_samples = 1
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=settings,
                allocator=_SupplyAllocator((("cpu",), ("cpu",))),
            )
            completed = {"gen0_peer0"}
            service.open_generation(0, deadline=time.time() + 600, cohort_size=2)
            service.configure_generation_maturity(
                0,
                cohort_size=2,
                mature_target=2,
                count_callback=lambda: len(completed),
                identity_callback=lambda: set(completed),
                distinct_peer_commitments=True,
            )
            for peer_id in ("gen0_peer0", "gen0_peer1"):
                service.register_idle_supply(peer_id, 0)
            first = HostSnapshot(8, 5, 5, 0, observed_at=1.0)
            service._refresh_mature_counts(first)
            service._reconcile_supply(first)
            self.assertIn("gen0_peer0", service._idle_supply_waiters)
            self.assertEqual(
                service.status()["resource_supply"]["leases"][0]["peer_id"],
                "gen0_peer1",
            )

            completed.add("gen0_peer1")
            second = HostSnapshot(8, 5, 5, 0, observed_at=2.0)
            service._refresh_mature_counts(second)
            service._reconcile_supply(second)
            leases = service.status()["resource_supply"]["leases"]
            self.assertEqual(len(leases), 1)
            self.assertEqual(leases[0]["peer_id"], "gen0_peer0")
            self.assertEqual(leases[0]["priority"], "frontier_followup")

    def test_assessment_rejects_ordinary_but_keeps_mature_topups_open(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=2),
                allocator=_GPUAllocator(""),
            )
            service.open_generation(0, deadline=time.time() + 600, cohort_size=4)
            service.begin_assessment(0, "evidence_ready")
            with self.assertRaisesRegex(ExperimentRejected, "only mature work"):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "experiment_id": "late-scout",
                        "work_class": "scout",
                    }
                )
            mature = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer1",
                    "generation_id": 0,
                    "experiment_id": "mature-topup",
                    "work_class": "mature",
                    "eta_seconds": 60,
                }
            )
            self.assertEqual(mature.state, "queued")
            with self.assertRaisesRegex(ExperimentRejected, "ETA does not fit"):
                service.submit(
                    {
                        "command": [sys.executable, "-c", "pass"],
                        "peer_id": "gen0_peer2",
                        "generation_id": 0,
                        "experiment_id": "unknown-duration-topup",
                        "work_class": "mature",
                    }
                )

    def test_same_basename_runs_have_distinct_resource_controller_owners(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            first = ExperimentSchedulerService(
                run_dir=Path(td) / "task-a" / "run",
                settings=_settings(maximum=1),
            )
            second = ExperimentSchedulerService(
                run_dir=Path(td) / "task-b" / "run",
                settings=_settings(maximum=1),
            )
            self.assertNotEqual(first.resource_owner_id, second.resource_owner_id)

    def test_removed_persisted_profile_fails_resume_instead_of_using_default(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            with self.assertRaisesRegex(RuntimeError, "no longer declared"):
                service._require_profile("removed-gpu-profile")

    def test_start_failure_closes_rpc_server_and_releases_owner(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.threading.Thread.start",
                    side_effect=OSError("thread unavailable"),
                ),
                self.assertRaisesRegex(OSError, "thread unavailable"),
            ):
                service.start()
            self.assertIsNone(service._server)
            self.assertIsNone(service._owner_lock_fd)
            self.assertFalse(service.endpoint.exists())

    def test_endpoint_publication_failure_never_starts_scheduler_threads(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.atomic_write_json",
                    side_effect=OSError("endpoint publication unavailable"),
                ),
                self.assertRaisesRegex(OSError, "endpoint publication unavailable"),
            ):
                service.start()
            self.assertFalse(service._server_thread.is_alive())
            self.assertFalse(service._worker_thread.is_alive())
            self.assertIsNone(service._server)
            self.assertIsNone(service._owner_lock_fd)
            self.assertFalse(service.endpoint.exists())
            self.assertFalse((service.state_dir / "endpoint.json").exists())

    def test_post_thread_start_failure_quiesces_worker_before_releasing_owner(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.logger.info",
                    side_effect=KeyboardInterrupt("startup interrupted"),
                ),
                self.assertRaisesRegex(KeyboardInterrupt, "startup interrupted"),
            ):
                service.start()
            self.assertFalse(service._server_thread.is_alive())
            self.assertFalse(service._worker_thread.is_alive())
            self.assertIsNone(service._server)
            self.assertIsNone(service._owner_lock_fd)
            self.assertFalse(service.endpoint.exists())
            self.assertFalse((service.state_dir / "endpoint.json").exists())

    def test_start_failure_retains_owner_for_recovered_active_processes(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            drained = threading.Event()

            def recover_active() -> None:
                service._active["recovered-live-job"] = MagicMock()

            def drain_active() -> None:
                service._active.clear()
                service._finalize_owner_resources()
                drained.set()

            with (
                patch.object(service, "_recover_active_process_groups", side_effect=recover_active),
                patch.object(service, "_worker", side_effect=drain_active),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler._UnixServer",
                    side_effect=OSError("socket unavailable"),
                ),
                self.assertRaisesRegex(OSError, "socket unavailable"),
            ):
                service.start()
            self.assertTrue(drained.wait(timeout=2))
            self.assertIsNone(service._owner_lock_fd)
            self.assertIsNone(service._server_thread)
            self.assertFalse(service._worker_thread.is_alive())

    def test_owner_metadata_failure_releases_file_lock(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            run_dir = Path(td) / "run"
            first = ExperimentSchedulerService(run_dir=run_dir, settings=_settings(maximum=1))
            with (
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.os.ftruncate",
                    side_effect=OSError("owner metadata unavailable"),
                ),
                self.assertRaisesRegex(OSError, "owner metadata unavailable"),
            ):
                first._acquire_owner_lock()
            self.assertIsNone(first._owner_lock_fd)

            second = ExperimentSchedulerService(run_dir=run_dir, settings=_settings(maximum=1))
            second._acquire_owner_lock()
            self.assertIsNotNone(second._owner_lock_fd)
            second._release_owner_lock()

    def test_start_cleanup_unlink_failure_does_not_strand_owner_lock(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            with (
                patch.object(Path, "unlink", side_effect=PermissionError("unlink unavailable")),
                self.assertRaisesRegex(PermissionError, "unlink unavailable"),
            ):
                service.start()
            self.assertIsNone(service._owner_lock_fd)

    def test_stop_retains_owner_until_transport_cleanup_succeeds(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            service._acquire_owner_lock()
            server = MagicMock()
            server.server_close.side_effect = OSError("server close unavailable")
            service._server = server
            with (
                self.assertRaisesRegex(OSError, "server close unavailable"),
            ):
                service.stop()
            self.assertIsNotNone(service._owner_lock_fd)
            self.assertFalse(service._transport_cleanup_complete)

            server.server_close.side_effect = None
            service.stop()
            self.assertIsNone(service._owner_lock_fd)
            self.assertTrue(service._transport_cleanup_complete)

    def test_failed_drainer_never_releases_owner_while_recovered_work_is_active(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )

            def recover_active() -> None:
                service._active["recovered-live-job"] = MagicMock()

            with (
                patch.object(service, "_recover_active_process_groups", side_effect=recover_active),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler._UnixServer",
                    side_effect=OSError("socket unavailable"),
                ),
                patch(
                    "praxist.plugins.workflow_stages.research_loop.backend."
                    "experiment_scheduler.threading.Thread.start",
                    side_effect=OSError("drainer unavailable"),
                ),
                self.assertRaisesRegex(OSError, "socket unavailable"),
            ):
                service.start()

            service.stop()
            self.assertIsNotNone(service._owner_lock_fd)
            service._active.clear()
            service.stop()
            self.assertIsNone(service._owner_lock_fd)

    def test_allocator_cleanup_failure_retains_owner_for_stop_retry(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _GPUAllocator("")
            allocator.close = MagicMock(side_effect=[OSError("controller unavailable"), None])
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )
            service._acquire_owner_lock()

            with self.assertRaisesRegex(RuntimeError, "cleanup remains pending"):
                service.stop()
            self.assertIsNotNone(service._owner_lock_fd)

            service.stop()
            self.assertIsNone(service._owner_lock_fd)
            self.assertEqual(allocator.close.call_count, 2)

    def test_worker_keeps_scheduler_observable_after_iteration_failure(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            allocator = _GPUAllocator("")
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=allocator,
            )

            def fail_once(*, queued: bool) -> HostSnapshot:
                del queued
                service._stopping = True
                service._transport_cleanup_complete = True
                raise RuntimeError("host observation failed")

            allocator.refresh = fail_once  # type: ignore[method-assign]
            with (
                patch.object(service._condition, "wait", return_value=None),
                patch.object(service, "_finalize_owner_resources"),
            ):
                service._worker()
            self.assertEqual(service._worker_error, "RuntimeError: host observation failed")

    def test_reconcile_retries_pending_submission_audit_without_losing_lease(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_SupplyAllocator((("cpu",),)),
            )
            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "pending-audit",
                }
            )
            lease = _SupplyLease(
                lease_id="supply-pending-audit",
                peer_id=job.peer_id,
                generation_id=job.generation_id,
                admissible_profiles=("cpu",),
                priority="mature",
                issued_at=time.time(),
                expires_at=time.time() + 60,
            )
            service._supply_leases[lease.lease_id] = lease
            job.supply_claim_id = lease.lease_id
            with patch.object(
                service,
                "_consume_pending_supply_locked",
                side_effect=OSError("audit unavailable"),
            ):
                service._reconcile_supply(HostSnapshot(8, 5, 5, 0))
            self.assertIn(lease.lease_id, service._supply_leases)
            self.assertEqual(job.supply_claim_id, lease.lease_id)

    def test_reconcile_drops_idle_waiter_after_generation_deadline(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            settings = _settings(maximum=1)
            settings.supply_idle_samples = 1
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=settings,
                allocator=_SupplyAllocator((("cpu",),)),
            )
            service.open_generation(0, deadline=time.time() - 1, cohort_size=1)
            service._idle_supply_waiters["gen0_peer0"] = (0, time.time())
            service._reconcile_supply(HostSnapshot(8, 5, 5, 0))
            self.assertNotIn("gen0_peer0", service._idle_supply_waiters)
            self.assertEqual(service.status()["resource_supply"]["leases"], [])

    def test_supply_replay_handles_stale_late_and_legacy_acceptance(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run", settings=_settings(maximum=1)
            )
            now = time.time()
            service._recover_supply_events(
                [
                    {"event": "supply_stale_submission", "priority": "mature"},
                    {
                        "event": "supply_consumed",
                        "lease_id": "terminal-without-grant",
                        "priority": "mature",
                    },
                    {
                        "event": "supply_granted",
                        "lease_id": "late",
                        "priority": "mature",
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "admissible_profiles": ["cpu"],
                        "issued_at": now - 20,
                        "expires_at": now - 10,
                    },
                    {
                        "event": "submitted",
                        "supply_lease_id": "late",
                        "peer_id": "gen0_peer0",
                        "generation_id": 0,
                        "profile": "cpu",
                        "work_class": "mature",
                        "recorded_at": now,
                    },
                    {
                        "event": "supply_granted",
                        "lease_id": "legacy-accepted",
                        "priority": "mature",
                        "peer_id": "gen0_peer1",
                        "generation_id": 0,
                        "admissible_profiles": ["cpu"],
                        "issued_at": now,
                        "expires_at": now + 60,
                    },
                    {
                        "event": "submitted",
                        "supply_lease_id": "legacy-accepted",
                        "peer_id": "gen0_peer1",
                        "generation_id": 0,
                        "profile": "cpu",
                        "work_class": "mature",
                        "recorded_at": now + 1,
                    },
                ]
            )
            status = service.status()["resource_supply"]["stats"]
            self.assertEqual(status["stale_submission"], 1)
            self.assertEqual(status["expired"], 1)
            self.assertEqual(status["consumed"], 1)
            self.assertIn("legacy-accepted", service._consumed_supply_leases)

    def test_recovery_helpers_preserve_live_identity_and_truncate_partial_audit(self) -> None:
        with tempfile.TemporaryDirectory() as td:
            service = ExperimentSchedulerService(
                run_dir=Path(td) / "run",
                settings=_settings(maximum=1),
                allocator=_GPUAllocator("GPU-0"),
            )
            service.allocator.snapshot = HostSnapshot(
                8,
                5,
                5,
                0,
                gpus=(GPUDevice(0, "GPU-0", 10000, 100, 2),),
                gpu_processes=(GPUProcess(999999, "GPU-0", 100, 2),),
            )
            with patch("os.getpgid", side_effect=OSError("process vanished")):
                self.assertEqual(
                    service._recovered_gpu_uuids({}, pid=999999, pgid=42),
                    ("GPU-0",),
                )
            self.assertEqual(service._recovered_profile({}, ("GPU-0",)).name, "gpu")

            job = service.submit(
                {
                    "command": [sys.executable, "-c", "pass"],
                    "peer_id": "gen0_peer0",
                    "generation_id": 0,
                    "experiment_id": "legacy-live",
                }
            )
            service._queue.clear()
            service._jobs.clear()
            service._semantic_jobs.clear()
            job.pid = os.getpid()
            job.pgid = os.getpgid(0)
            service._adopt_unaccounted_live(job, {}, reason="legacy_process")
            self.assertIn(job.job_id, service._active)

            audit = Path(td) / "partial.jsonl"
            audit.write_bytes(b"complete\n" + b"x" * 70000)
            fd = os.open(audit, os.O_RDWR)
            try:
                service._truncate_incomplete_event_tail(fd)
            finally:
                os.close(fd)
            self.assertEqual(audit.read_bytes(), b"complete\n")
            self.assertFalse(service._event_line_present(Path(td) / "missing", b"event\n"))


if __name__ == "__main__":
    unittest.main()