Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

27.6 KB
Raw Blame
"""Tests for the PG-queue consumer prefork supervisor (UN-3606).

DB-free / fork-free: the env knob, the ``_Fleet`` bookkeeping, the reap/restart
scheduling, the join+SIGKILL escalation and the fork/health guards are exercised in
isolation (``os.*`` mocked) so the suite stays fast and deterministic — no real
children, no worker bootstrap, no signals installed.
"""

import errno
import threading
import time
from unittest.mock import MagicMock, patch

import pytest

from pg_queue_consumer import supervisor as sup
from pg_queue_consumer.supervisor import (
    _CRASH_LOOP_THRESHOLD,
    _DEFAULT_SHUTDOWN_GRACE_SECONDS,
    _MAX_CONCURRENCY,
    _MIN_BOOTSTRAP_BUDGET_SECONDS,
    _MIN_HEALTHY_UPTIME_SECONDS,
    _Fleet,
    _child_after_fork,
    _join_children,
    _kill_stale_children,
    _monitor_tick,
    _reap_dead,
    _restart_due_children,
    _run_child,
    _try_fork_child,
    _wait_for_exit,
    child_watchdog_from_env,
    concurrency_from_env,
    shutdown_grace_from_env,
)

_MOD = "pg_queue_consumer.supervisor"
_ENV = "WORKER_PG_QUEUE_CONSUMER_CONCURRENCY"


_VT = "WORKER_PG_QUEUE_CONSUMER_VT_SECONDS"
_GRACE = "WORKER_PG_QUEUE_CONSUMER_SHUTDOWN_GRACE_SECONDS"


class TestShutdownGraceFromEnv:
    """UN-3695: the shutdown drain grace must track the consumer VT (not a hardcoded
    30s), so a SIGTERM drains an in-flight batch instead of SIGKILLing it mid-flight.
    """

    def test_unset_vt_and_override_defaults_to_fallback(self, monkeypatch):
        monkeypatch.delenv(_VT, raising=False)
        monkeypatch.delenv(_GRACE, raising=False)
        assert shutdown_grace_from_env() == pytest.approx(_DEFAULT_SHUTDOWN_GRACE_SECONDS)

    def test_tracks_vt_when_set(self, monkeypatch):
        # The fileproc chart sets VT=9060 → grace must follow it, not stay at 30.
        monkeypatch.delenv(_GRACE, raising=False)
        monkeypatch.setenv(_VT, "9060")
        assert shutdown_grace_from_env() == pytest.approx(9060.0)

    def test_vt_below_fallback_is_floored(self, monkeypatch):
        monkeypatch.delenv(_GRACE, raising=False)
        monkeypatch.setenv(_VT, "10")
        assert shutdown_grace_from_env() == pytest.approx(_DEFAULT_SHUTDOWN_GRACE_SECONDS)

    def test_explicit_override_wins_over_vt(self, monkeypatch):
        monkeypatch.setenv(_VT, "9060")
        monkeypatch.setenv(_GRACE, "120")
        assert shutdown_grace_from_env() == pytest.approx(120.0)

    def test_override_honoured_as_is_even_below_fallback(self, monkeypatch):
        # An explicit short dev drain is respected (not floored) — only the VT path floors.
        monkeypatch.delenv(_VT, raising=False)
        monkeypatch.setenv(_GRACE, "5")
        assert shutdown_grace_from_env() == pytest.approx(5.0)

    def test_negative_override_raises(self, monkeypatch):
        # A negative drain budget would collapse to 0 → SIGKILL with no drain (the
        # orphan bug this module prevents). Must fail fast, not silently.
        monkeypatch.setenv(_GRACE, "-5")
        with pytest.raises(ValueError, match="SHUTDOWN_GRACE_SECONDS"):
            shutdown_grace_from_env()

    def test_non_finite_override_raises(self, monkeypatch):
        # inf would make the join deadline now+inf → shutdown hangs until k8s
        # hard-kills the pod; nan collapses to 0. Reject both.
        monkeypatch.setenv(_GRACE, "inf")
        with pytest.raises(ValueError, match="finite"):
            shutdown_grace_from_env()

    def test_malformed_override_raises(self, monkeypatch):
        monkeypatch.setenv(_GRACE, "abc")
        with pytest.raises(ValueError, match="SHUTDOWN_GRACE_SECONDS"):
            shutdown_grace_from_env()

    def test_malformed_vt_raises(self, monkeypatch):
        # Fail-fast-at-startup contract: a bad VT surfaces the offending var name.
        monkeypatch.delenv(_GRACE, raising=False)
        monkeypatch.setenv(_VT, "9060s")
        with pytest.raises(ValueError, match="VT_SECONDS"):
            shutdown_grace_from_env()


class TestConcurrencyFromEnv:
    def test_unset_defaults_to_one(self, monkeypatch):
        monkeypatch.delenv(_ENV, raising=False)
        assert concurrency_from_env() == 1

    def test_empty_defaults_to_one(self, monkeypatch):
        monkeypatch.setenv(_ENV, "")
        assert concurrency_from_env() == 1

    def test_valid_value(self, monkeypatch):
        monkeypatch.setenv(_ENV, "4")
        assert concurrency_from_env() == 4

    def test_clamped_to_max(self, monkeypatch):
        monkeypatch.setenv(_ENV, str(_MAX_CONCURRENCY + 50))
        assert concurrency_from_env() == _MAX_CONCURRENCY

    @pytest.mark.parametrize("bad", ["0", "-3"])
    def test_below_one_raises(self, monkeypatch, bad):
        monkeypatch.setenv(_ENV, bad)
        with pytest.raises(ValueError, match=">= 1"):
            concurrency_from_env()

    def test_non_int_raises(self, monkeypatch):
        monkeypatch.setenv(_ENV, "abc")
        with pytest.raises(ValueError, match="Invalid"):
            concurrency_from_env()


class TestFleet:
    def test_record_and_reap_keep_structures_consistent(self):
        f = _Fleet(2)
        f.record_fork(0, 111)
        assert f.alive_items() == [(0, 111)] and f.alive_count() == 1
        uptime = f.reap(0)
        assert uptime >= 0  # a non-negative monotonic duration
        assert f.alive_count() == 0  # pid + last_fork dropped together

    def test_slot_out_of_range_raises(self):
        f = _Fleet(2)
        with pytest.raises(IndexError):
            f.record_fork(5, 111)

    def test_record_fork_does_not_reseed_heartbeat(self):
        # The crash-loop fix: a re-fork must NOT refresh the slot, or a child that
        # never polls looks perpetually fresh.
        f = _Fleet(1)
        f._heartbeats[0] = time.time() - 500  # an aged slot
        f.record_fork(0, 111)
        assert f.oldest_age() > 400  # still aged, not reset to ~0

    def test_immediate_crash_increments_then_loops(self):
        f = _Fleet(1)
        for i in range(1, _CRASH_LOOP_THRESHOLD + 1):
            n = f.schedule_restart(0, uptime=0.1)  # died immediately
            assert n == i
        assert f.is_crash_looping() is True

    def test_healthy_uptime_resets_crash_counter(self):
        f = _Fleet(1)
        f.schedule_restart(0, uptime=0.1)
        assert f.schedule_restart(0, uptime=0.1) == 2
        n = f.schedule_restart(0, uptime=_MIN_HEALTHY_UPTIME_SECONDS + 1)
        assert n == 0 and f.is_crash_looping() is False

    def test_freshness_is_inf_when_crash_looping(self):
        import math

        f = _Fleet(1)
        for _ in range(_CRASH_LOOP_THRESHOLD):
            f.schedule_restart(0, uptime=0.1)
        assert math.isinf(f.freshness())

    def test_freshness_is_oldest_age_when_healthy(self):
        f = _Fleet(2)
        f._heartbeats[1] = time.time() - 100
        assert 99 < f.freshness() < 102

    def test_due_restarts_respects_backoff(self, monkeypatch):
        f = _Fleet(1)
        clock = [1000.0]
        monkeypatch.setattr(f"{_MOD}.time.monotonic", lambda: clock[0])
        f.schedule_restart(0, uptime=0.1)  # backoff scheduled in the future
        assert f.due_restarts() == []  # not yet due
        clock[0] += 100.0  # well past any backoff
        assert f.due_restarts() == [0]

    # --- readiness (UN-4136) ---------------------------------------------------

    def test_fresh_fleet_is_not_loaded(self):
        # Unlike the heartbeats (seeded fresh), nothing is loaded at construction:
        # a pod must not go Ready before its children have imported.
        f = _Fleet(3)
        assert f.loaded_count() == 0
        assert f.all_loaded() is False

    def test_all_loaded_only_when_every_slot_has_loaded(self):
        f = _Fleet(3)
        f.loaded[0] = 1
        f.loaded[2] = 1
        assert f.loaded_count() == 2
        assert f.all_loaded() is False  # slot 1 still importing
        f.loaded[1] = 1
        assert f.all_loaded() is True

    def test_reap_clears_the_slot_loaded_flag(self):
        # A dead child's replacement must bootstrap again before the fleet counts
        # as loaded; the stale flag must not carry over.
        f = _Fleet(2)
        f.loaded[0] = f.loaded[1] = 1
        f.record_fork(1, 111)
        f.reap(1)
        assert list(f.loaded) == [1, 0]
        assert f.all_loaded() is False


_STALE = "WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS"
_WATCHDOG = "WORKER_PG_QUEUE_CONSUMER_CHILD_WATCHDOG"


class TestChildWatchdogFromEnv:
    """UN-4223: opt-in, because it turns HEALTH_STALE into a hard per-task cap —
    which docker-compose (no healthcheck) has never enforced.
    """

    def test_off_by_default_even_with_health_stale(self, monkeypatch):
        monkeypatch.delenv(_WATCHDOG, raising=False)
        monkeypatch.setenv(_STALE, "3720")
        assert child_watchdog_from_env() is None

    @pytest.mark.parametrize("on", ["true", "1", "yes", "ON"])
    def test_enabled_tracks_health_stale(self, monkeypatch, on):
        monkeypatch.setenv(_WATCHDOG, on)
        monkeypatch.setenv(_STALE, "7260")
        assert child_watchdog_from_env() == pytest.approx(7260.0)

    @pytest.mark.parametrize("off", ["false", "0", "no", "OFF"])
    def test_explicitly_off(self, monkeypatch, off):
        monkeypatch.setenv(_STALE, "7260")
        monkeypatch.setenv(_WATCHDOG, off)
        assert child_watchdog_from_env() is None

    def test_enabled_without_health_stale_raises(self, monkeypatch):
        # The 60s code default would kill any task longer than a minute.
        monkeypatch.setenv(_WATCHDOG, "true")
        monkeypatch.delenv(_STALE, raising=False)
        with pytest.raises(ValueError, match=_STALE):
            child_watchdog_from_env()

    def test_malformed_switch_raises(self, monkeypatch):
        monkeypatch.setenv(_STALE, "600")
        monkeypatch.setenv(_WATCHDOG, "maybe")
        with pytest.raises(ValueError, match=_WATCHDOG):
            child_watchdog_from_env()

    @pytest.mark.parametrize("bad", ["0", "-5", "inf", "nan"])
    def test_non_positive_or_non_finite_stale_raises(self, monkeypatch, bad):
        monkeypatch.setenv(_WATCHDOG, "true")
        monkeypatch.setenv(_STALE, bad)
        with pytest.raises(ValueError, match=_STALE):
            child_watchdog_from_env()


class TestKillStaleChildren:
    """UN-4223: one hung child is killed on its own instead of a liveness kill
    draining every sibling with it.
    """

    @staticmethod
    def _fleet(ages: list[float], loaded: list[int]) -> _Fleet:
        f = _Fleet(len(ages))
        now = time.time()
        for slot, age in enumerate(ages):
            f._heartbeats[slot] = now - age
            f.loaded[slot] = loaded[slot]
            f.record_fork(slot, 100 + slot)
        return f

    def test_only_the_stale_child_is_sigkilled(self):
        import signal as _signal

        f = self._fleet([5, 9000, 5], loaded=[1, 1, 1])
        killed: set[int] = set()
        with patch(f"{_MOD}.os.kill") as kill:
            _kill_stale_children(f, stale_after=7260, killed=killed)
        kill.assert_called_once_with(101, _signal.SIGKILL)
        assert killed == {101}

    def test_bootstrapping_child_is_judged_by_fork_age_not_heartbeat(self):
        # A re-forked slot keeps its predecessor's old heartbeat until it loads,
        # so a recent fork is spared however stale that heartbeat looks.
        f = self._fleet([9000], loaded=[0])
        with patch(f"{_MOD}.os.kill") as kill:
            _kill_stale_children(f, stale_after=7260, killed=set())
        kill.assert_not_called()

    def test_child_hung_in_bootstrap_is_killed(self):
        import signal as _signal

        f = self._fleet([0], loaded=[0])
        f._last_fork[0] -= 7300  # forked long ago, never loaded
        with patch(f"{_MOD}.os.kill") as kill:
            _kill_stale_children(f, stale_after=7260, killed=set())
        kill.assert_called_once_with(100, _signal.SIGKILL)

    def test_slow_bootstrap_gets_at_least_the_bootstrap_floor(self):
        # A short task threshold (ide-callback: 180s) must not cut off an import
        # that is merely slow, e.g. under a CPU cap.
        f = self._fleet([0], loaded=[0])
        f._last_fork[0] -= _MIN_BOOTSTRAP_BUDGET_SECONDS - 60
        with patch(f"{_MOD}.os.kill") as kill:
            _kill_stale_children(f, stale_after=180, killed=set())
        kill.assert_not_called()
        f._last_fork[0] -= 120  # now past the floor
        with patch(f"{_MOD}.os.kill") as kill:
            _kill_stale_children(f, stale_after=180, killed=set())
        kill.assert_called_once()

    def test_age_at_threshold_is_spared(self):
        f = self._fleet([60], loaded=[1])
        with (
            patch(f"{_MOD}.time.time", return_value=f._heartbeats[0] + 60),
            patch(f"{_MOD}.os.kill") as kill,
        ):
            _kill_stale_children(f, stale_after=60, killed=set())
        kill.assert_not_called()

    def test_already_signalled_child_is_not_killed_again(self):
        f = self._fleet([9000], loaded=[1])
        with patch(f"{_MOD}.os.kill") as kill:
            _kill_stale_children(f, stale_after=7260, killed={100})
        kill.assert_not_called()

    def test_vanished_child_is_tolerated_and_not_counted(self):
        f = self._fleet([9000], loaded=[1])
        killed: set[int] = set()
        with patch(f"{_MOD}.os.kill", side_effect=ProcessLookupError):
            _kill_stale_children(f, stale_after=7260, killed=killed)
        assert killed == {100}  # not signalled again before the reap
        assert f.watchdog_kills == 0  # the metric counts kills that happened

    def test_kill_is_counted_for_metrics(self):
        f = self._fleet([9000, 9000], loaded=[1, 1])
        with patch(f"{_MOD}.os.kill"):
            _kill_stale_children(f, stale_after=7260, killed=set())
        assert f.watchdog_kills == 2


class TestMonitorTick:
    """The loop wiring: reap, trim ``killed``, re-fork, then the watchdog."""

    @staticmethod
    def _stale_fleet() -> _Fleet:
        f = _Fleet(1)
        f._heartbeats[0] = time.time() - 9000
        f.loaded[0] = 1
        f.record_fork(0, 100)
        return f

    def test_watchdog_runs_when_enabled(self):
        f = self._stale_fleet()
        with (
            patch(f"{_MOD}.os.waitpid", return_value=(0, 0)),
            patch(f"{_MOD}.os.kill") as kill,
        ):
            _monitor_tick(f, threading.Event(), 7260, set())
        kill.assert_called_once()

    def test_watchdog_skipped_when_disabled(self):
        f = self._stale_fleet()
        with (
            patch(f"{_MOD}.os.waitpid", return_value=(0, 0)),
            patch(f"{_MOD}.os.kill") as kill,
        ):
            _monitor_tick(f, threading.Event(), None, set())
        kill.assert_not_called()

    def test_watchdog_skipped_while_stopping(self):
        # Draining children must not be SIGKILLed during shutdown.
        f = self._stale_fleet()
        stopping = threading.Event()
        stopping.set()
        with (
            patch(f"{_MOD}.os.waitpid", return_value=(0, 0)),
            patch(f"{_MOD}.os.kill") as kill,
        ):
            _monitor_tick(f, stopping, 7260, set())
        kill.assert_not_called()

    def test_reaped_pid_leaves_killed_and_slot_is_scheduled(self):
        f = self._stale_fleet()
        f._last_fork[0] -= _MIN_HEALTHY_UPTIME_SECONDS + 100
        killed = {100}
        with (
            patch(f"{_MOD}.os.waitpid", return_value=(100, 9)),
            patch(f"{_MOD}.os.kill"),
            patch(f"{_MOD}._try_fork_child"),
        ):
            _monitor_tick(f, threading.Event(), 7260, killed)
        assert killed == set()  # a recycled pid can't inherit the mark
        assert 0 in f._restart_due  # re-fork scheduled
        assert f._consecutive_crashes[0] == 0  # a long-lived child, not a crash


class TestReapDead:
    def test_dead_child_reaped_and_rescheduled(self):
        f = _Fleet(1)
        f.record_fork(0, 111)
        stopping = threading.Event()
        with patch(f"{_MOD}.os.waitpid", return_value=(111, 0)):  # exited
            _reap_dead(f, stopping)
        assert f.alive_count() == 0
        assert f.due_restarts() in ([], [0])  # scheduled (maybe backed off)
        assert f._consecutive_crashes.get(0, 0) >= 1  # counted (immediate death)

    def test_live_child_left_alone(self):
        f = _Fleet(1)
        f.record_fork(0, 111)
        with patch(f"{_MOD}.os.waitpid", return_value=(0, 0)):  # alive
            _reap_dead(f, threading.Event())
        assert f.alive_items() == [(0, 111)]

    def test_not_rescheduled_while_stopping(self):
        f = _Fleet(1)
        f.record_fork(0, 111)
        stopping = threading.Event()
        stopping.set()
        with patch(f"{_MOD}.os.waitpid", return_value=(111, 0)):
            _reap_dead(f, stopping)
        assert f.alive_count() == 0  # reaped
        assert f.due_restarts() == []  # but NOT scheduled for restart

    def test_already_reaped_is_treated_gone(self):
        f = _Fleet(1)
        f.record_fork(0, 111)
        with patch(f"{_MOD}.os.waitpid", side_effect=ChildProcessError()):
            _reap_dead(f, threading.Event())
        assert f.alive_count() == 0


class TestRestartDueChildren:
    def test_due_slot_is_reforked(self):
        f = _Fleet(1)
        f.schedule_restart(0, uptime=0.1)
        with (
            patch(f"{_MOD}.time.monotonic", return_value=1e9),  # everything due
            patch(f"{_MOD}._try_fork_child") as fork,
        ):
            _restart_due_children(f, threading.Event())
        fork.assert_called_once_with(f, 0)

    def test_stopping_blocks_refork(self):
        f = _Fleet(1)
        f.schedule_restart(0, uptime=0.1)
        stopping = threading.Event()
        stopping.set()
        with (
            patch(f"{_MOD}.time.monotonic", return_value=1e9),
            patch(f"{_MOD}._try_fork_child") as fork,
        ):
            _restart_due_children(f, stopping)
        fork.assert_not_called()  # no fresh child spawned into shutdown


class TestTryForkChild:
    def test_fork_oserror_returns_false_and_does_not_record(self):
        f = _Fleet(1)
        with patch(f"{_MOD}.os.fork", side_effect=OSError(errno.EAGAIN, "again")):
            assert _try_fork_child(f, 0) is False
        assert f.alive_count() == 0  # slot left for the next monitor tick

    def test_parent_records_child(self):
        f = _Fleet(1)
        with patch(f"{_MOD}.os.fork", return_value=222):  # parent sees child pid
            assert _try_fork_child(f, 0) is True
        assert f.alive_items() == [(0, 222)]

    def test_child_is_handed_the_shared_heartbeat_and_loaded_arrays(self):
        # The child writes its own slot in BOTH arrays; handing it a copy (or not
        # handing over `loaded` at all) would leave /ready stuck at 503.
        f = _Fleet(1)
        with (
            patch(f"{_MOD}.os.fork", return_value=0),  # we are the child
            patch(f"{_MOD}._child_after_fork", side_effect=SystemExit) as child,
        ):
            with pytest.raises(SystemExit):
                _try_fork_child(f, 0)
        child.assert_called_once_with(0, f.heartbeats, f.loaded)


class TestChildAfterFork:
    def test_resets_signals_and_exits_zero_on_clean_run(self):
        with (
            patch(f"{_MOD}.signal.signal") as sig,
            patch(f"{_MOD}._run_child") as run,
            patch(f"{_MOD}.os._exit", side_effect=SystemExit) as exit_,
        ):
            heartbeats, loaded = MagicMock(), MagicMock()
            with pytest.raises(SystemExit):
                _child_after_fork(0, heartbeats, loaded)
        # SIGTERM + SIGINT reset to default before running.
        assert sig.call_count == 2
        run.assert_called_once_with(0, heartbeats, loaded)
        exit_.assert_called_once_with(0)

    def test_hard_exits_one_when_run_raises(self):
        with (
            patch(f"{_MOD}.signal.signal"),
            patch(f"{_MOD}._run_child", side_effect=RuntimeError("boom")),
            patch(f"{_MOD}.os._exit", side_effect=SystemExit) as exit_,
        ):
            with pytest.raises(SystemExit):
                _child_after_fork(0, MagicMock(), MagicMock())
        exit_.assert_called_once_with(1)


class TestRunChildLoaded:
    """The child marks itself loaded only after the bootstrap, and before it
    starts polling (UN-4136).
    """

    @staticmethod
    def _run_slot_1(fleet: _Fleet, build) -> None:  # noqa: ANN001
        # No real `import worker` bootstrap and no real heartbeat thread.
        with (
            patch.dict("sys.modules", {"worker": MagicMock()}),
            patch("pg_queue_consumer._bootstrap.select_source_worker_type"),
            patch(
                "queue_backend.pg_queue.consumer.build_consumer_from_env",
                side_effect=build,
            ),
            patch(f"{_MOD}.threading.Thread"),
        ):
            _run_child(1, fleet.heartbeats, fleet.loaded)

    def test_marks_its_slot_loaded_before_polling(self):
        f = _Fleet(2)
        seen: list[list[int]] = []
        consumer = MagicMock()
        # Snapshot the flags at the moment polling would begin.
        consumer.run.side_effect = lambda: seen.append(list(f.loaded))
        self._run_slot_1(f, lambda: consumer)
        assert seen == [[0, 1]]  # own slot only, and set before run()

    def test_heartbeat_is_fresh_by_the_time_the_slot_is_loaded(self):
        # UN-4223: the publisher thread is patched out here, so only the
        # synchronous publish can refresh the predecessor's stale heartbeat.
        f = _Fleet(2)
        f._heartbeats[1] = time.time() - 9000
        consumer = MagicMock()
        consumer.seconds_since_last_poll.return_value = 0.0
        ages: list[float] = []
        consumer.run.side_effect = lambda: ages.append(f.slot_age(1))
        self._run_slot_1(f, lambda: consumer)
        assert len(ages) == 1
        assert ages[0] < 5

    def test_not_marked_loaded_when_the_build_fails(self):
        # A child that cannot finish its bootstrap must never count as loaded.
        f = _Fleet(2)

        def _boom():  # noqa: ANN202
            raise RuntimeError("cannot build")

        with pytest.raises(RuntimeError, match="cannot build"):
            self._run_slot_1(f, _boom)
        assert list(f.loaded) == [0, 0]


class TestWaitForExit:
    def test_true_when_child_exits(self):
        with patch(f"{_MOD}.os.waitpid", return_value=(111, 0)):
            assert _wait_for_exit(111, time.monotonic() + 5) is True

    def test_true_when_already_reaped(self):
        with patch(f"{_MOD}.os.waitpid", side_effect=ChildProcessError()):
            assert _wait_for_exit(111, time.monotonic() + 5) is True

    def test_false_when_deadline_passes(self):
        with patch(f"{_MOD}.os.waitpid", return_value=(0, 0)):  # never exits
            assert _wait_for_exit(111, time.monotonic() - 1) is False  # already past


class TestJoinChildren:
    def test_clean_exit_no_sigkill(self):
        f = _Fleet(1)
        f.record_fork(0, 111)
        with (
            patch(f"{_MOD}._wait_for_exit", return_value=True),
            patch(f"{_MOD}.os.kill") as kill,
        ):
            _join_children(f, grace_seconds=5)
        kill.assert_not_called()

    def test_straggler_is_sigkilled_and_reaped(self):
        f = _Fleet(1)
        f.record_fork(0, 111)
        with (
            patch(f"{_MOD}._wait_for_exit", return_value=False),  # never drained
            patch(f"{_MOD}.os.kill") as kill,
            patch(f"{_MOD}.os.waitpid") as waitpid,
        ):
            _join_children(f, grace_seconds=5)
        import signal as _signal

        kill.assert_called_once_with(111, _signal.SIGKILL)
        waitpid.assert_called_once_with(111, 0)

    def test_shared_deadline_not_per_child(self):
        # UN-3695 / debate H2: all children waited against ONE shared deadline (a
        # per-child deadline serializes N wedged children to N×grace and blows past
        # the pod terminationGracePeriodSeconds → siblings hard-killed). Advancing
        # time.monotonic pins it: shared → the deadline is computed once (105 for
        # all three); a per-child recompute would yield 105/205/305.
        f = _Fleet(3)
        for slot, pid in enumerate((111, 222, 333)):
            f.record_fork(slot, pid)
        deadlines: list[float] = []

        def _record_drained(_pid, deadline):
            deadlines.append(deadline)
            return True  # child exited within the window

        with (
            patch(f"{_MOD}.time.monotonic", side_effect=[100.0, 200.0, 300.0, 400.0]),
            patch(f"{_MOD}._wait_for_exit", side_effect=_record_drained),
        ):
            _join_children(f, grace_seconds=5)
        assert deadlines == pytest.approx([105.0, 105.0, 105.0])  # computed once, shared

    def test_multi_child_stragglers_share_deadline_and_all_sigkilled(self):
        # 3 wedged children (none drain) → each SIGKILLed + reaped against the SAME
        # shared deadline. Kill-count alone wouldn't catch a per-child regression,
        # so also assert the deadline handed to every child was identical.
        f = _Fleet(3)
        for slot, pid in enumerate((111, 222, 333)):
            f.record_fork(slot, pid)
        seen: list[float] = []

        def _record_wedged(_pid, deadline):
            seen.append(deadline)
            return False  # never drains → forces SIGKILL

        with (
            patch(f"{_MOD}._wait_for_exit", side_effect=_record_wedged),
            patch(f"{_MOD}.os.kill") as kill,
            patch(f"{_MOD}.os.waitpid") as waitpid,
        ):
            _join_children(f, grace_seconds=5)
        assert kill.call_count == 3
        assert waitpid.call_count == 3
        assert len(set(seen)) == 1  # one shared deadline across all three


class TestSupervisorHealth:
    def test_no_port_returns_none(self, monkeypatch):
        monkeypatch.delenv("WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT", raising=False)
        assert sup._maybe_start_supervisor_health(_Fleet(1)) is None

    def test_bind_error_is_swallowed(self, monkeypatch):
        monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT", "8090")
        server = MagicMock()
        server.start.side_effect = OSError(errno.EADDRINUSE, "in use")
        # LivenessServer is imported locally inside the function → patch its source.
        with patch(
            "queue_backend.pg_queue.liveness.LivenessServer", return_value=server
        ):
            # Must not propagate — the consumer keeps draining without a probe.
            assert sup._maybe_start_supervisor_health(_Fleet(1)) is None

    def test_ready_is_503_until_every_child_loads_while_health_stays_200(
        self, monkeypatch
    ):
        # UN-4136: the whole point. /health is fresh from construction (liveness
        # must not trip during the import), so only /ready can hold the pod
        # NotReady while the children import.
        import json
        import urllib.error
        import urllib.request

        monkeypatch.setenv("WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT", "0")  # OS picks
        fleet = _Fleet(2)
        server = sup._maybe_start_supervisor_health(fleet)
        assert server is not None
        base = f"http://127.0.0.1:{server.bound_port}"

        def _ready() -> tuple[int, dict]:
            try:
                with urllib.request.urlopen(f"{base}/ready", timeout=5) as resp:
                    return resp.status, json.loads(resp.read())
            except urllib.error.HTTPError as exc:
                return exc.code, json.loads(exc.read())

        try:
            with urllib.request.urlopen(f"{base}/health", timeout=5) as resp:
                assert resp.status == 200  # alive, but...

            code, body = _ready()
            assert (code, body["status"], body["loaded_children"]) == (503, "starting", 0)

            fleet.loaded[0] = 1
            code, body = _ready()
            assert (code, body["loaded_children"]) == (503, 1)  # one still importing

            fleet.loaded[1] = 1
            code, body = _ready()
            assert (code, body["status"], body["loaded_children"]) == (200, "ready", 2)
        finally:
            server.stop()