Skip to content

Self-improving AI Employees for everyone

feat(supervisor): BaseSupervisor stack for binary / sidecar lifecycle

trohitgaadf616083ad
diff --git a/server/services/_supervisor/__init__.py b/server/services/_supervisor/__init__.py
new file mode 100644
index 00000000..5ea83c5b
--- /dev/null
+++ b/server/services/_supervisor/__init__.py
@@ -0,0 +1,36 @@
+"""Common base classes for supervised binaries and sidecars.
+
+Use :class:`BaseProcessSupervisor` for services that own a child process
+and :class:`BaseClientSupervisor` for HTTP-only or in-process-task
+services. Both inherit from :class:`BaseSupervisor` which provides the
+uniform start/stop/status surface.
+
+See ``docs-internal/supervisor.md`` (TBD) for the migration guide.
+"""
+
+from .base import BaseSupervisor, RestartPolicy
+from .client import BaseClientSupervisor
+from .process import BaseProcessSupervisor
+from .registry import (
+ get_supervisor,
+ list_supervisors,
+ register_supervisor,
+ shutdown_all_supervisors,
+ status_snapshot_all,
+)
+from .util import drain_stream, kill_tree, terminate_then_kill
+
+__all__ = [
+ "BaseSupervisor",
+ "BaseProcessSupervisor",
+ "BaseClientSupervisor",
+ "RestartPolicy",
+ "drain_stream",
+ "kill_tree",
+ "terminate_then_kill",
+ "register_supervisor",
+ "get_supervisor",
+ "list_supervisors",
+ "shutdown_all_supervisors",
+ "status_snapshot_all",
+]
diff --git a/server/services/_supervisor/base.py b/server/services/_supervisor/base.py
new file mode 100644
index 00000000..3117ac83
--- /dev/null
+++ b/server/services/_supervisor/base.py
@@ -0,0 +1,153 @@
+"""Abstract base for any supervised binary, sidecar, or in-process task.
+
+Every subclass owns one logical resource (a child process, an HTTP
+client, a polling task) and exposes the same lifecycle surface so the
+DI container, FastAPI lifespan, and StatusBroadcaster can treat all
+supervisors uniformly.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any, ClassVar, Optional
+
+import anyio
+from tenacity import AsyncRetrying, stop_after_attempt, wait_random_exponential
+
+from core.logging import get_logger
+
+
+@dataclass(frozen=True)
+class RestartPolicy:
+ """Pluggable restart policy translated to ``tenacity.AsyncRetrying``.
+
+ Default values mirror tenacity's own defaults so a bare
+ ``RestartPolicy()`` is sane.
+ """
+
+ attempts: int = 3
+ min_wait: float = 1.0
+ max_wait: float = 30.0
+
+
+class BaseSupervisor(ABC):
+ """Shared lifecycle scaffolding for supervisors."""
+
+ # Subclasses set this if they want a non-class-name label in status
+ # snapshots and log lines.
+ name: Optional[str] = None
+
+ # Classvar singleton storage so subclasses can use the uniform
+ # `get_instance()` accessor. Each subclass gets its own slot.
+ _instance: ClassVar[Optional["BaseSupervisor"]] = None
+
+ def __init__(self) -> None:
+ self._lock = anyio.Lock()
+ self._started_at: Optional[datetime] = None
+ self._last_error: Optional[str] = None
+ self._logger = get_logger(self.__class__.__module__)
+
+ # ---- singleton accessor ---------------------------------------------
+
+ @classmethod
+ def get_instance(cls, *args, **kwargs) -> "BaseSupervisor":
+ """Return the per-subclass singleton, constructing on first call.
+
+ ``*args`` / ``**kwargs`` are forwarded to ``__init__`` and only
+ consulted on the first call. Plugins typically wrap this in a
+ module-level ``get_xxx_runtime()`` helper so call sites stay tidy.
+ """
+ if cls._instance is None:
+ cls._instance = cls(*args, **kwargs)
+ return cls._instance
+
+ @classmethod
+ async def reset_instance(cls) -> None:
+ """Stop and discard the singleton (useful for tests)."""
+ if cls._instance is not None:
+ try:
+ if cls._instance.is_running():
+ await cls._instance.stop()
+ finally:
+ cls._instance = None
+
+ # ---- public lifecycle ------------------------------------------------
+
+ @property
+ def label(self) -> str:
+ return self.name or self.__class__.__name__
+
+ async def start(self) -> None:
+ """Idempotent start. Subclass implements ``_do_start``."""
+ async with self._lock:
+ if self.is_running():
+ return
+ try:
+ await self._do_start()
+ self._started_at = datetime.now(timezone.utc)
+ self._last_error = None
+ except Exception as exc:
+ self._last_error = str(exc)
+ raise
+
+ async def stop(self) -> None:
+ """Idempotent stop. Subclass implements ``_do_stop``."""
+ async with self._lock:
+ if not self.is_running():
+ return
+ try:
+ await self._do_stop()
+ finally:
+ self._started_at = None
+
+ async def restart(self, *, policy: Optional[RestartPolicy] = None) -> None:
+ """Stop then start. With ``policy``, retries the start with backoff."""
+ await self.stop()
+ if policy is None:
+ await self.start()
+ return
+ async for attempt in AsyncRetrying(
+ stop=stop_after_attempt(policy.attempts),
+ wait=wait_random_exponential(min=policy.min_wait, max=policy.max_wait),
+ reraise=True,
+ ):
+ with attempt:
+ await self.start()
+
+ def status_snapshot(self) -> dict[str, Any]:
+ """Uniform shape for the broadcaster / health endpoints."""
+ return {
+ "name": self.label,
+ "running": self.is_running(),
+ "started_at": self._started_at.isoformat() if self._started_at else None,
+ "last_error": self._last_error,
+ **self._extra_status(),
+ }
+
+ # ---- subclass surface ------------------------------------------------
+
+ @abstractmethod
+ def is_running(self) -> bool:
+ """Return True iff the supervised resource is currently alive."""
+
+ @abstractmethod
+ async def _do_start(self) -> None:
+ """Subclass spawn / connect logic."""
+
+ @abstractmethod
+ async def _do_stop(self) -> None:
+ """Subclass teardown logic."""
+
+ async def health_check(self) -> bool:
+ """Readiness probe. Default: trust ``is_running()``.
+
+ Subclasses with cheap probes (TCP connect, HTTP /health) should
+ override.
+ """
+ return self.is_running()
+
+ def _extra_status(self) -> dict[str, Any]:
+ """Subclass status fields (pid, port, etc.). Empty by default."""
+ return {}
diff --git a/server/services/_supervisor/client.py b/server/services/_supervisor/client.py
new file mode 100644
index 00000000..febff021
--- /dev/null
+++ b/server/services/_supervisor/client.py
@@ -0,0 +1,55 @@
+"""Client / in-process-task supervisor.
+
+For services that don't own a subprocess but still have a lifecycle:
+HTTP clients pointed at an externally-spawned daemon (nodejs_client),
+or in-process tasks like the python-telegram-bot polling loop.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+import aiohttp
+
+from .base import BaseSupervisor
+
+
+class BaseClientSupervisor(BaseSupervisor):
+ """A :class:`BaseSupervisor` that owns an aiohttp session.
+
+ Subclasses with non-HTTP lifecycles (e.g. polling tasks) override
+ ``_do_start`` / ``_do_stop`` / ``is_running`` directly and don't have
+ to use ``_session`` at all.
+ """
+
+ # Subclasses with an HTTP backend set this; pure-task subclasses
+ # leave it empty and override ``health_check``.
+ base_url: str = ""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self._session: Optional[aiohttp.ClientSession] = None
+
+ def is_running(self) -> bool:
+ return self._session is not None and not self._session.closed
+
+ async def _do_start(self) -> None:
+ if self._session is None or self._session.closed:
+ self._session = aiohttp.ClientSession()
+
+ async def _do_stop(self) -> None:
+ if self._session is not None and not self._session.closed:
+ await self._session.close()
+ self._session = None
+
+ async def health_check(self) -> bool:
+ if not self.base_url or not self.is_running():
+ return self.is_running()
+ try:
+ async with self._session.get(f"{self.base_url.rstrip('/')}/health") as resp:
+ return 200 <= resp.status < 300
+ except aiohttp.ClientError:
+ return False
+
+ def _extra_status(self) -> dict[str, Any]:
+ return {"base_url": self.base_url or None}
diff --git a/server/services/_supervisor/process.py b/server/services/_supervisor/process.py
new file mode 100644
index 00000000..44304a79
--- /dev/null
+++ b/server/services/_supervisor/process.py
@@ -0,0 +1,125 @@
+"""Subprocess-owning supervisor.
+
+Subclass for any service that spawns and supervises a long-lived child
+process (Go binary, Node sidecar, etc.). One-shot CLI wrappers should
+inherit from :class:`BaseSupervisor` directly and use the helpers in
+``_supervisor.util`` instead.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import subprocess
+import sys
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any, Optional
+
+import anyio
+
+from .base import BaseSupervisor
+from .util import drain_stream, terminate_then_kill
+
+
+class BaseProcessSupervisor(BaseSupervisor):
+ """A :class:`BaseSupervisor` that owns one ``anyio.abc.Process``."""
+
+ # Inherit stdio (False) is the simplest and matches the WhatsApp
+ # default — the child's logs go straight to the backend console.
+ # Set True when the subclass wants line-by-line drain into the
+ # Python logger (e.g. process_service for log files + broadcast).
+ pipe_streams: bool = False
+
+ # Time budget between graceful-stop and forceful tree-kill.
+ terminate_grace_seconds: float = 5.0
+
+ # Windows-only: spawn the child in a new process group so we can
+ # send CTRL_BREAK_EVENT on shutdown. No-op on POSIX.
+ graceful_shutdown: bool = False
+
+ def __init__(self) -> None:
+ super().__init__()
+ self._proc: Optional[anyio.abc.Process] = None
+ self._drain_tasks: list[asyncio.Task] = []
+
+ # ---- subclass surface ------------------------------------------------
+
+ @abstractmethod
+ def binary_path(self) -> Path:
+ """Locate the binary on disk (shutil.which / env override / etc.)."""
+
+ @abstractmethod
+ def argv(self) -> list[str]:
+ """Full argv for the child process (binary + flags)."""
+
+ def cwd(self) -> Optional[Path]:
+ """Override to set the child's working directory."""
+ return None
+
+ def env(self) -> dict[str, str]:
+ """Override to add to the child's environment. Defaults to inherit."""
+ return {**os.environ}
+
+ async def _pre_spawn(self) -> None:
+ """Hook before ``anyio.open_process`` (e.g., write a config file)."""
+
+ # ---- lifecycle implementation ---------------------------------------
+
+ def is_running(self) -> bool:
+ return self._proc is not None and self._proc.returncode is None
+
+ async def _do_start(self) -> None:
+ binary = self.binary_path()
+ if not binary.exists():
+ raise FileNotFoundError(f"{self.label} binary not found at {binary}")
+
+ await self._pre_spawn()
+
+ argv = self.argv()
+ kwargs: dict[str, Any] = {
+ "cwd": str(self.cwd()) if self.cwd() else None,
+ "env": self.env(),
+ }
+ if self.pipe_streams:
+ kwargs["stdout"] = subprocess.PIPE
+ kwargs["stderr"] = subprocess.PIPE
+
+ # Windows-only: opt into a new process group for graceful CTRL_BREAK.
+ if sys.platform == "win32" and self.graceful_shutdown:
+ kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore[attr-defined]
+
+ self._proc = await anyio.open_process(argv, **kwargs)
+ self._logger.info(
+ "[%s] spawned pid=%s binary=%s",
+ self.label, self._proc.pid, binary,
+ )
+
+ if self.pipe_streams:
+ self._drain_tasks = [
+ asyncio.create_task(
+ drain_stream(self._proc.stdout, self._logger.info, prefix=f"[{self.label}] "),
+ ),
+ asyncio.create_task(
+ drain_stream(self._proc.stderr, self._logger.error, prefix=f"[{self.label}] "),
+ ),
+ ]
+
+ async def _do_stop(self) -> None:
+ proc = self._proc
+ self._proc = None
+ if proc is None:
+ return
+ pid = proc.pid
+ await terminate_then_kill(
+ proc,
+ grace=self.terminate_grace_seconds,
+ use_ctrl_break=self.graceful_shutdown,
+ )
+ for task in self._drain_tasks:
+ task.cancel()
+ self._drain_tasks = []
+ self._logger.info("[%s] stopped pid=%s", self.label, pid)
+
+ def _extra_status(self) -> dict[str, Any]:
+ return {"pid": self._proc.pid if self._proc else None}
diff --git a/server/services/_supervisor/registry.py b/server/services/_supervisor/registry.py
new file mode 100644
index 00000000..715a87e2
--- /dev/null
+++ b/server/services/_supervisor/registry.py
@@ -0,0 +1,63 @@
+"""Cross-plugin registry of live supervisors.
+
+Plugin packages register their supervisor singletons here so the FastAPI
+lifespan can stop them all in one shot, and the broadcaster/health
+endpoints can enumerate their status without hardcoded per-plugin
+imports.
+
+Same pattern as ``services.ws_handler_registry``,
+``services.event_waiter.FILTER_BUILDERS``, and
+``services.status_broadcaster._SERVICE_REFRESH_CALLBACKS``.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Dict, List
+
+from .base import BaseSupervisor
+
+logger = logging.getLogger(__name__)
+
+_SUPERVISORS: Dict[str, BaseSupervisor] = {}
+
+
+def register_supervisor(supervisor: BaseSupervisor) -> None:
+ """Add a supervisor to the registry (idempotent on label collision)."""
+ label = supervisor.label
+ existing = _SUPERVISORS.get(label)
+ if existing is supervisor:
+ return
+ if existing is not None:
+ raise ValueError(
+ f"Supervisor label '{label}' already registered "
+ f"({existing.__class__.__name__} vs {supervisor.__class__.__name__})"
+ )
+ _SUPERVISORS[label] = supervisor
+
+
+def list_supervisors() -> List[BaseSupervisor]:
+ return list(_SUPERVISORS.values())
+
+
+def get_supervisor(label: str) -> BaseSupervisor | None:
+ return _SUPERVISORS.get(label)
+
+
+async def shutdown_all_supervisors() -> None:
+ """Stop every registered supervisor. Best-effort, errors logged."""
+ for supervisor in list_supervisors():
+ if not supervisor.is_running():
+ continue
+ try:
+ await supervisor.stop()
+ except Exception as exc:
+ logger.warning(
+ "[supervisor] %s shutdown failed: %s",
+ supervisor.label, exc,
+ )
+
+
+def status_snapshot_all() -> List[dict]:
+ """Status snapshots for every registered supervisor."""
+ return [s.status_snapshot() for s in list_supervisors()]
diff --git a/server/services/_supervisor/util.py b/server/services/_supervisor/util.py
new file mode 100644
index 00000000..e6b9e61e
--- /dev/null
+++ b/server/services/_supervisor/util.py
@@ -0,0 +1,115 @@
+"""Cross-platform helpers for supervised binaries.
+
+Stdlib + psutil only — no exotic abstractions. These are extracted from
+the patterns already in production use across browser_service.py,
+process_service.py, and the WhatsApp runtime, so behaviour is unchanged
+when subclasses adopt the supervisor base.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import signal
+import sys
+from typing import Awaitable, Callable, Optional
+
+import anyio
+import psutil
+
+
+def kill_tree(pid: int) -> None:
+ """Kill a process and all descendants. Cross-platform via psutil.
+
+ Defensively guards every psutil call against ``NoSuchProcess`` to
+ survive races with fast-exiting children. Same shape as
+ ``process_service.py:_kill_process_tree`` and
+ ``browser_service.py:_kill_process_tree``.
+ """
+ try:
+ parent = psutil.Process(pid)
+ except psutil.NoSuchProcess:
+ return
+ try:
+ children = parent.children(recursive=True)
+ except psutil.NoSuchProcess:
+ children = []
+ for child in children:
+ try:
+ child.kill()
+ except psutil.NoSuchProcess:
+ pass
+ try:
+ parent.kill()
+ except psutil.NoSuchProcess:
+ pass
+
+
+async def terminate_then_kill(
+ proc: anyio.abc.Process,
+ *,
+ grace: float = 5.0,
+ use_ctrl_break: bool = False,
+) -> None:
+ """Send a graceful stop, wait ``grace`` seconds, then tree-kill.
+
+ POSIX path: ``proc.terminate()`` (SIGTERM), wait, ``kill_tree()``.
+ Windows graceful path (``use_ctrl_break=True``, requires the process
+ to have been spawned with ``CREATE_NEW_PROCESS_GROUP``):
+ ``os.kill(pid, CTRL_BREAK_EVENT)`` — the only way to send a real
+ SIGINT-equivalent on Windows.
+ """
+ if proc.returncode is not None:
+ return
+
+ if use_ctrl_break and sys.platform == "win32":
+ try:
+ os.kill(proc.pid, signal.CTRL_BREAK_EVENT)
+ except (ProcessLookupError, OSError):
+ pass
+ else:
+ try:
+ proc.terminate()
+ except ProcessLookupError:
+ pass
+
+ with anyio.move_on_after(grace):
+ await proc.wait()
+
+ if proc.returncode is None:
+ kill_tree(proc.pid)
+ await proc.wait()
+
+
+async def drain_stream(
+ stream: Optional[anyio.abc.ByteReceiveStream],
+ log_fn: Callable[[str], None],
+ *,
+ prefix: str = "",
+) -> None:
+ """Forward subprocess output line-by-line to a logger callable.
+
+ Safe against cancellation and closed streams. ``log_fn`` is something
+ like ``logger.info`` or ``logger.error``. Lines are decoded as UTF-8
+ with ``errors="replace"`` so binary garbage never crashes the drain.
+ """
+ if stream is None:
+ return
+ buf = b""
+ try:
+ async for chunk in stream:
+ buf += chunk
+ while b"\n" in buf:
+ line, buf = buf.split(b"\n", 1)
+ text = line.decode("utf-8", errors="replace").rstrip()
+ if text:
+ log_fn(f"{prefix}{text}" if prefix else text)
+ if buf:
+ text = buf.decode("utf-8", errors="replace").rstrip()
+ if text:
+ log_fn(f"{prefix}{text}" if prefix else text)
+ except (anyio.ClosedResourceError, anyio.EndOfStream, asyncio.CancelledError):
+ pass
+ except Exception as exc: # pragma: no cover — defensive
+ logging.getLogger(__name__).debug("drain_stream ended: %s", exc)
diff --git a/server/tests/services/__init__.py b/server/tests/services/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/server/tests/services/test_supervisor.py b/server/tests/services/test_supervisor.py
new file mode 100644
index 00000000..849bce73
--- /dev/null
+++ b/server/tests/services/test_supervisor.py
@@ -0,0 +1,286 @@
+"""Tests for the BaseSupervisor / BaseProcessSupervisor / BaseClientSupervisor stack."""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import anyio
+import pytest
+
+# Ensure server/ on sys.path (mirrors conftest.py)
+SERVER_DIR = Path(__file__).resolve().parents[2]
+if str(SERVER_DIR) not in sys.path:
+ sys.path.insert(0, str(SERVER_DIR))
+
+from services._supervisor import ( # noqa: E402
+ BaseClientSupervisor,
+ BaseProcessSupervisor,
+ BaseSupervisor,
+ RestartPolicy,
+ get_supervisor,
+ list_supervisors,
+ register_supervisor,
+ shutdown_all_supervisors,
+)
+from services._supervisor.registry import _SUPERVISORS # noqa: E402
+from services._supervisor.util import drain_stream, kill_tree, terminate_then_kill # noqa: E402
+
+
+# --- helpers ---------------------------------------------------------------
+
+
+class _FakeSupervisor(BaseSupervisor):
+ """Minimal concrete supervisor that flips a `running` flag."""
+
+ def __init__(self, name: str = "fake") -> None:
+ super().__init__()
+ self.name = name
+ self._running = False
+ self.start_calls = 0
+ self.stop_calls = 0
+ self.fail_next_start = False
+
+ def is_running(self) -> bool:
+ return self._running
+
+ async def _do_start(self) -> None:
+ self.start_calls += 1
+ if self.fail_next_start:
+ self.fail_next_start = False
+ raise RuntimeError("boom")
+ self._running = True
+
+ async def _do_stop(self) -> None:
+ self.stop_calls += 1
+ self._running = False
+
+
[email protected](autouse=True)
+def _clear_registry():
+ """Each test starts with an empty supervisor registry + reset singletons."""
+ _SUPERVISORS.clear()
+ yield
+ _SUPERVISORS.clear()
+ # Wipe per-class singletons to avoid cross-test bleed
+ for klass in (BaseSupervisor, BaseProcessSupervisor, BaseClientSupervisor, _FakeSupervisor):
+ klass._instance = None
+
+
+# --- BaseSupervisor lifecycle ----------------------------------------------
+
+
+async def test_start_stop_idempotent():
+ s = _FakeSupervisor()
+ await s.start()
+ await s.start() # second call is no-op (lock + is_running guard)
+ assert s.start_calls == 1
+ assert s.is_running()
+
+ await s.stop()
+ await s.stop()
+ assert s.stop_calls == 1
+ assert not s.is_running()
+
+
+async def test_start_failure_records_last_error():
+ s = _FakeSupervisor()
+ s.fail_next_start = True
+ with pytest.raises(RuntimeError, match="boom"):
+ await s.start()
+ assert s._last_error == "boom"
+ assert not s.is_running()
+
+
+async def test_status_snapshot_shape():
+ s = _FakeSupervisor(name="hello")
+ snap = s.status_snapshot()
+ assert snap["name"] == "hello"
+ assert snap["running"] is False
+ assert snap["started_at"] is None
+ assert snap["last_error"] is None
+
+ await s.start()
+ snap = s.status_snapshot()
+ assert snap["running"] is True
+ assert snap["started_at"] is not None
+
+
+async def test_restart_with_policy_retries():
+ s = _FakeSupervisor()
+ await s.start()
+ s.fail_next_start = True # first restart attempt fails
+ policy = RestartPolicy(attempts=3, min_wait=0.0, max_wait=0.0)
+ await s.restart(policy=policy)
+ # Stop once, then start failed, then start succeeded
+ assert s.stop_calls == 1
+ assert s.start_calls >= 2
+ assert s.is_running()
+
+
+# --- Singleton classmethod -------------------------------------------------
+
+
+def test_get_instance_returns_same_singleton():
+ a = _FakeSupervisor.get_instance(name="solo")
+ b = _FakeSupervisor.get_instance() # ignored args after first
+ assert a is b
+ assert a.name == "solo"
+
+
+async def test_reset_instance_clears_singleton():
+ a = _FakeSupervisor.get_instance(name="first")
+ await _FakeSupervisor.reset_instance()
+ b = _FakeSupervisor.get_instance(name="second")
+ assert a is not b
+ assert b.name == "second"
+
+
+# --- Registry --------------------------------------------------------------
+
+
+def test_register_and_list_supervisors():
+ s1 = _FakeSupervisor(name="alpha")
+ s2 = _FakeSupervisor(name="beta")
+ register_supervisor(s1)
+ register_supervisor(s2)
+ register_supervisor(s1) # idempotent on same instance
+ assert {s.label for s in list_supervisors()} == {"alpha", "beta"}
+ assert get_supervisor("alpha") is s1
+
+
+def test_register_collision_raises():
+ register_supervisor(_FakeSupervisor(name="dup"))
+ with pytest.raises(ValueError, match="already registered"):
+ register_supervisor(_FakeSupervisor(name="dup"))
+
+
+async def test_shutdown_all_supervisors_stops_running_only():
+ s1 = _FakeSupervisor(name="r")
+ s2 = _FakeSupervisor(name="s")
+ register_supervisor(s1)
+ register_supervisor(s2)
+ await s1.start()
+ # s2 stays unstarted
+ await shutdown_all_supervisors()
+ assert s1.stop_calls == 1
+ assert s2.stop_calls == 0
+
+
+async def test_shutdown_all_supervisors_swallows_errors():
+ class _BadStop(_FakeSupervisor):
+ async def _do_stop(self) -> None:
+ raise RuntimeError("explode")
+
+ s = _BadStop(name="bad")
+ register_supervisor(s)
+ await s.start()
+ await shutdown_all_supervisors() # must not raise
+
+
+# --- util.kill_tree --------------------------------------------------------
+
+
+def test_kill_tree_no_such_process_is_noop():
+ # Non-existent PID — should not raise.
+ kill_tree(999_999_999)
+
+
+def test_kill_tree_walks_descendants():
+ fake_child = MagicMock()
+ fake_parent = MagicMock()
+ fake_parent.children.return_value = [fake_child]
+
+ with patch("services._supervisor.util.psutil.Process", return_value=fake_parent):
+ kill_tree(123)
+
+ fake_child.kill.assert_called_once()
+ fake_parent.kill.assert_called_once()
+
+
+# --- util.terminate_then_kill ---------------------------------------------
+
+
+async def test_terminate_then_kill_returns_early_if_already_dead():
+ proc = MagicMock()
+ proc.returncode = 0 # already exited
+ await terminate_then_kill(proc)
+ proc.terminate.assert_not_called()
+
+
+async def test_terminate_then_kill_force_kills_after_grace():
+ proc = MagicMock()
+ proc.returncode = None # appears alive
+ proc.pid = 42
+ proc.wait = AsyncMock()
+ # First wait (in move_on_after) does nothing; then we force-kill.
+ proc.wait.side_effect = [
+ asyncio.sleep(10), # exceeds grace
+ None, # post-kill wait succeeds
+ ]
+ proc.terminate = MagicMock()
+
+ with patch("services._supervisor.util.kill_tree") as kill_tree_mock:
+ await terminate_then_kill(proc, grace=0.05)
+
+ proc.terminate.assert_called_once()
+ kill_tree_mock.assert_called_once_with(42)
+
+
+# --- util.drain_stream -----------------------------------------------------
+
+
+async def test_drain_stream_handles_none():
+ captured = []
+ await drain_stream(None, captured.append)
+ assert captured == []
+
+
+async def test_drain_stream_forwards_lines_with_prefix():
+ class _FakeStream:
+ def __init__(self, chunks):
+ self._chunks = chunks
+
+ def __aiter__(self):
+ self._iter = iter(self._chunks)
+ return self
+
+ async def __anext__(self):
+ try:
+ return next(self._iter)
+ except StopIteration:
+ raise StopAsyncIteration
+
+ captured = []
+ stream = _FakeStream([b"hello\n", b"world\n", b"trailing"])
+ await drain_stream(stream, captured.append, prefix="[x] ")
+ assert captured == ["[x] hello", "[x] world", "[x] trailing"]
+
+
+# --- BaseClientSupervisor defaults ----------------------------------------
+
+
+async def test_client_supervisor_session_lifecycle():
+ class _C(BaseClientSupervisor):
+ name = "client"
+
+ c = _C()
+ assert not c.is_running()
+ await c.start()
+ assert c.is_running()
+ await c.stop()
+ assert not c.is_running()