Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

34.9 KB
Raw Blame
"""Helpers for isolating and terminating managed child processes."""

from __future__ import annotations

import asyncio
import hashlib
import hmac
import logging
import math
import os
import secrets
import signal
import subprocess
import time
from dataclasses import dataclass
from typing import Any, Callable, Literal

import psutil

KILL_SIGNAL = getattr(signal, "SIGKILL", signal.SIGTERM)
PROCESS_IDENTITY_ENV = "AVIBE_PROCESS_IDENTITY"
DEFAULT_PROCESS_TERMINATE_TIMEOUT_SECONDS = 3.0

#: What a liveness probe could actually establish about a pid: a process is there,
#: nothing is there, or this pass could not tell. Only the middle answer means a
#: durable handle on that process has stopped being worth keeping.
ProcessLivenessStatus = Literal["alive", "gone", "unknown"]

#: What a recovery reap established about a managed tree: this pass stopped it, it
#: was already gone, or neither could be shown. Only the last one keeps the record.
ProcessReapOutcome = Literal["reaped", "gone", "unconfirmed"]

#: What a LOOK established about a managed tree, with nothing signalled: something of
#: it is still there, it is provably gone, or this pass could not tell. A caller
#: deciding whether it may start a second copy of that work must treat the last two
#: differently -- only "gone" is permission.
ProcessTreePresence = Literal["present", "gone", "unknown"]


@dataclass(frozen=True)
class ProcessIdentity:
    pid: int
    create_time: float
    worker_fingerprint: str | None
    # False when the environment could not be read: the marker is unknown, which
    # is not the same as absent (``worker_fingerprint`` is then ``None`` too).
    marker_readable: bool = True


@dataclass(frozen=True)
class PersistedProcessIdentity:
    pid: int
    create_time: float
    worker_fingerprint: str


def new_process_identity_marker() -> str:
    """Return a high-entropy marker inherited by a managed process tree."""
    return secrets.token_hex(32)


def fingerprint_process_marker(marker: str) -> str:
    return f"sha256:{hashlib.sha256(marker.encode('ascii')).hexdigest()}"


def process_identity_subprocess_env(marker: str) -> dict[str, str]:
    env = dict(os.environ)
    env[PROCESS_IDENTITY_ENV] = marker
    return env


def capture_spawned_process_identity(
    pid: int,
    marker: str,
    *,
    on_error: Callable[[BaseException], None] | None = None,
) -> PersistedProcessIdentity | None:
    if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
        return None
    try:
        create_time = float(psutil.Process(pid).create_time())
    except (psutil.Error, OSError, TypeError, ValueError, OverflowError) as exc:
        # Callers may retain a safe diagnostic without changing the fail-closed
        # identity result or gaining authority to signal an unverified pid.
        if on_error is not None:
            on_error(exc)
        return None
    if not math.isfinite(create_time) or create_time <= 0:
        return None
    return PersistedProcessIdentity(
        pid=pid,
        create_time=create_time,
        worker_fingerprint=fingerprint_process_marker(marker),
    )


def serialize_process_identity(identity: PersistedProcessIdentity) -> dict[str, Any]:
    """The JSON form of a captured identity, for any store that persists one."""

    return {
        "pid": identity.pid,
        "create_time": identity.create_time,
        "worker_fingerprint": identity.worker_fingerprint,
    }


def is_valid_worker_fingerprint(value: Any) -> bool:
    if not isinstance(value, str) or not value.startswith("sha256:") or len(value) != 71:
        return False
    return all(char in "0123456789abcdef" for char in value[7:])


def process_identity_from_payload(
    payload: Any,
    pid: int,
) -> PersistedProcessIdentity | None:
    """Rebuild a persisted identity, or ``None`` if the payload cannot be trusted.

    Deliberately strict, because the ONLY consumer of the result is a kill: a
    payload that has been truncated, hand-edited, or written by an older format
    must produce ``None`` (leave the process alone) rather than a plausible-looking
    identity that authorises signalling some unrelated pid.
    """

    if not isinstance(payload, dict):
        return None
    identity_pid = payload.get("pid")
    create_time = payload.get("create_time")
    worker_fingerprint = payload.get("worker_fingerprint")
    try:
        normalized_create_time = float(create_time)
    except (TypeError, ValueError, OverflowError):
        return None
    if (
        not isinstance(identity_pid, int)
        or isinstance(identity_pid, bool)
        or identity_pid != pid
        or not isinstance(create_time, (int, float))
        or isinstance(create_time, bool)
        or not math.isfinite(normalized_create_time)
        or normalized_create_time <= 0
        or not is_valid_worker_fingerprint(worker_fingerprint)
    ):
        return None
    return PersistedProcessIdentity(
        pid=identity_pid,
        create_time=normalized_create_time,
        worker_fingerprint=worker_fingerprint,
    )


def _marker_matches(expected: PersistedProcessIdentity, live: ProcessIdentity) -> bool:
    return live.worker_fingerprint is not None and hmac.compare_digest(
        expected.worker_fingerprint,
        live.worker_fingerprint,
    )


def process_identity_matches(
    expected: PersistedProcessIdentity,
    live: ProcessIdentity,
) -> bool:
    """Whether ``live`` is the recorded process, by its inherited marker.

    The marker is the authority, not the birth time: on macOS psutil's displayed
    create time can shift while the same execution keeps running, and a 256-bit
    marker only the managed tree inherits cannot be carried by a stranger holding
    a recycled pid.
    """

    return expected.pid == live.pid and _marker_matches(expected, live)


def process_identity_recycled(expected: PersistedProcessIdentity, live: ProcessIdentity) -> bool:
    """Whether the recorded pid now provably belongs to another process.

    Only a different birth time AND a readable environment without the matching
    marker says so; a marker match wins over a shifted birth time, and an
    unreadable marker proves nothing either way.
    """

    return live.marker_readable and live.create_time != expected.create_time and not _marker_matches(expected, live)


def _open_process_identity(pid: int) -> tuple[psutil.Process, ProcessIdentity]:
    process = psutil.Process(pid)
    create_time = float(process.create_time())
    worker_fingerprint = None
    marker_readable = True
    try:
        marker = process.environ().get(PROCESS_IDENTITY_ENV)
    except (psutil.Error, OSError):
        marker = None
        marker_readable = False
    if isinstance(marker, str) and marker:
        worker_fingerprint = fingerprint_process_marker(marker)
    return process, ProcessIdentity(
        pid=pid,
        create_time=create_time,
        worker_fingerprint=worker_fingerprint,
        marker_readable=marker_readable,
    )


def inspect_process_identity(pid: int) -> ProcessIdentity | None:
    """Return the birth identity and inherited marker for a live process."""
    if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
        return None
    try:
        _process, identity = _open_process_identity(pid)
    except (psutil.Error, OSError, ValueError):
        return None
    return identity


def probe_process_liveness(pid: int) -> ProcessLivenessStatus:
    """Say whether a pid is occupied, free, or unanswerable right now.

    ``inspect_process_identity`` collapses the third answer into ``None``, and for
    its own callers that is correct: they are deciding whether they may still
    *steer* a process, and a pid they cannot read is one they must not signal
    either way. A caller deciding whether to DISCARD the only durable handle on
    that process needs the answers apart. "Gone" retires the record because there
    is nothing left to kill; "unanswerable" -- an exhausted fd table, a
    ``/proc`` read losing a race, a platform refusing ``create_time`` -- says
    only that this pass could not look, and throwing the record away then makes a
    still-running backup unkillable by every pass after it too.

    Mirrors ``process_group_identity_status``: the same three-way verdict, at the
    granularity of a single pid rather than a group.
    """

    if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
        # Not a pid at all, so there is no process to preserve a handle for.
        return "gone"
    try:
        _open_process_identity(pid)
    except (psutil.NoSuchProcess, ProcessLookupError):
        return "gone"
    except (psutil.Error, OSError, ValueError):
        return "unknown"
    return "alive"


def isolated_subprocess_kwargs(*, console: bool = True) -> dict[str, Any]:
    """Return subprocess kwargs that put a child outside this process group.

    ``console=False`` is for a console program that only ever runs in the
    background. On Windows it then gets no console window, which a parent with
    no console of its own -- the desktop app starts the service that way --
    would otherwise open for it. Desktop-managed processes always use this
    policy, including callers that leave ``console`` at its CLI default.
    """
    if os.name == "nt":
        flags = subprocess.CREATE_NEW_PROCESS_GROUP
        desktop_managed = any(
            os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
            for name in ("AVIBE_DESKTOP_SHELL", "AVIBE_DESKTOP_MANAGED_RUNTIME")
        )
        if not console or desktop_managed:
            flags |= subprocess.CREATE_NO_WINDOW
        return {"creationflags": flags}
    return {"start_new_session": True}


def _safe_signal_known_process_group(
    pgid: int,
    pid: int,
    sig: int,
    logger: logging.Logger,
    label: str,
    *,
    expected_identity: PersistedProcessIdentity | None = None,
    require_leader: bool = True,
) -> bool:
    if os.name == "nt" or not hasattr(os, "getpgid") or not hasattr(os, "killpg"):
        return False
    try:
        own_pgid = os.getpgrp()
    except Exception:
        logger.debug("Failed to inspect the service process group while signaling %s pid=%s", label, pid, exc_info=True)
        return False
    if pgid == own_pgid:
        logger.error(
            "Refusing to signal %s process group for pid=%s because it matches the avibe service pgid=%s",
            label,
            pid,
            own_pgid,
        )
        return False
    if expected_identity is not None:
        if require_leader:
            try:
                _process, live_identity = _open_process_identity(expected_identity.pid)
                live_pgid = os.getpgid(expected_identity.pid)
            except (ProcessLookupError, psutil.NoSuchProcess):
                logger.warning(
                    "Refusing to signal %s process group pgid=%s because its supervisor exited",
                    label,
                    pgid,
                )
                return False
            except Exception:
                logger.debug(
                    "Failed to revalidate %s process group pgid=%s before signaling",
                    label,
                    pgid,
                    exc_info=True,
                )
                return False
            if live_pgid != pgid or not process_identity_matches(expected_identity, live_identity):
                logger.warning(
                    "Refusing to signal %s process group pgid=%s because its supervisor identity changed",
                    label,
                    pgid,
                )
                return False
        elif not process_group_matches_identity(pgid, expected_identity, logger, label):
            logger.warning(
                "Refusing to signal %s process group pgid=%s without a matching supervisor marker",
                label,
                pgid,
            )
            return False
    try:
        logger.info(
            "Signaling %s process group pgid=%s pid=%s signal=%s service_pgid=%s",
            label,
            pgid,
            pid,
            sig,
            own_pgid,
        )
        os.killpg(pgid, sig)
        return True
    except ProcessLookupError:
        return True
    except Exception:
        logger.debug("Failed to signal %s process group pgid=%s", label, pgid, exc_info=True)
        return False


def _safe_signal_process_group(pid: int, sig: int, logger: logging.Logger, label: str) -> bool:
    if os.name == "nt" or not hasattr(os, "getpgid") or not hasattr(os, "killpg"):
        return False
    try:
        pgid = os.getpgid(pid)
    except ProcessLookupError:
        return True
    except Exception:
        logger.debug("Failed to inspect process group for %s pid=%s", label, pid, exc_info=True)
        return False
    return _safe_signal_known_process_group(pgid, pid, sig, logger, label)


def signal_process_tree(process: Any, sig: int, logger: logging.Logger, label: str) -> None:
    """Signal a managed process group, falling back to the direct process."""
    pid = getattr(process, "pid", None)
    if isinstance(pid, int) and _safe_signal_process_group(pid, sig, logger, label):
        return

    try:
        logger.info("Signaling %s direct process pid=%s signal=%s", label, pid, sig)
        if sig == signal.SIGTERM:
            process.terminate()
        elif sig == KILL_SIGNAL:
            process.kill()
        else:
            process.send_signal(sig)
    except ProcessLookupError:
        return


def _process_group_exists(pgid: int, logger: logging.Logger, label: str) -> bool:
    if os.name == "nt" or not hasattr(os, "killpg"):
        return False
    try:
        os.killpg(pgid, 0)
    except ProcessLookupError:
        return False
    except PermissionError:
        return True
    except Exception:
        logger.debug("Failed to inspect %s process group pgid=%s", label, pgid, exc_info=True)
        return True
    return True


def process_group_exists(pgid: int, logger: logging.Logger, label: str) -> bool:
    """Return whether an isolated POSIX process group still has members."""
    if not isinstance(pgid, int) or isinstance(pgid, bool) or pgid <= 0:
        return False
    return _process_group_exists(pgid, logger, label)


def process_group_identity_status(
    pgid: int,
    expected_identity: PersistedProcessIdentity,
    logger: logging.Logger,
    label: str,
) -> str:
    """Return match, mismatch, or unknown for a persisted process group."""
    if (
        os.name == "nt"
        or not isinstance(pgid, int)
        or isinstance(pgid, bool)
        or pgid <= 0
        or not isinstance(expected_identity, PersistedProcessIdentity)
    ):
        return "unknown"
    saw_marker = False
    saw_unverified_member = False
    try:
        processes = psutil.process_iter(["pid"])
        for process in processes:
            pid = process.info.get("pid")
            if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
                continue
            try:
                if os.getpgid(pid) != pgid:
                    continue
            except (ProcessLookupError, PermissionError):
                continue
            except Exception:
                logger.debug(
                    "Failed to inspect %s process group member pid=%s",
                    label,
                    pid,
                    exc_info=True,
                )
                continue
            live_identity = inspect_process_identity(pid)
            if live_identity is not None and _marker_matches(expected_identity, live_identity):
                return "match"
            if (
                live_identity is None
                or live_identity.create_time < expected_identity.create_time
                or live_identity.worker_fingerprint is None
            ):
                saw_unverified_member = True
                continue
            saw_marker = True
    except Exception:
        logger.debug(
            "Failed to enumerate %s process group pgid=%s",
            label,
            pgid,
            exc_info=True,
        )
    return "mismatch" if saw_marker and not saw_unverified_member else "unknown"


def process_group_matches_identity(
    pgid: int,
    expected_identity: PersistedProcessIdentity,
    logger: logging.Logger,
    label: str,
) -> bool:
    return (
        process_group_identity_status(
            pgid,
            expected_identity,
            logger,
            label,
        )
        == "match"
    )


def _wait_for_process_group_exit(
    pgid: int,
    logger: logging.Logger,
    label: str,
    *,
    timeout: float,
) -> bool:
    deadline = time.monotonic() + max(0.0, timeout)
    while _process_group_exists(pgid, logger, label):
        remaining = deadline - time.monotonic()
        if remaining <= 0:
            return False
        time.sleep(min(0.1, remaining))
    return True


def terminate_process_group_by_pgid(
    pgid: int,
    logger: logging.Logger,
    label: str,
    *,
    expected_identity: PersistedProcessIdentity,
    terminate_timeout: float = DEFAULT_PROCESS_TERMINATE_TIMEOUT_SECONDS,
) -> bool:
    """Terminate a persisted isolated group after its original leader exited."""
    if not isinstance(pgid, int) or isinstance(pgid, bool) or pgid <= 0:
        return False
    if os.name == "nt":
        return False
    try:
        own_pgid = os.getpgrp()
    except Exception:
        logger.debug("Failed to inspect the service process group before recovering %s", label, exc_info=True)
        return False
    if pgid == own_pgid:
        logger.error(
            "Refusing to terminate %s process group because pgid=%s matches the avibe service",
            label,
            pgid,
        )
        return False
    if not _process_group_exists(pgid, logger, label):
        return True
    if not _safe_signal_known_process_group(
        pgid,
        expected_identity.pid,
        signal.SIGTERM,
        logger,
        label,
        expected_identity=expected_identity,
        require_leader=False,
    ):
        return False
    if _wait_for_process_group_exit(pgid, logger, label, timeout=terminate_timeout):
        return True

    logger.warning("Escalating termination of %s process group pgid=%s", label, pgid)
    if not _safe_signal_known_process_group(
        pgid,
        expected_identity.pid,
        KILL_SIGNAL,
        logger,
        label,
        expected_identity=expected_identity,
        require_leader=False,
    ):
        return False
    if _wait_for_process_group_exit(pgid, logger, label, timeout=terminate_timeout):
        return True
    logger.error("%s process group pgid=%s survived forced termination", label, pgid)
    return False


def _windows_process_tree(process: psutil.Process) -> list[psutil.Process]:
    try:
        children = process.children(recursive=True)
    except psutil.NoSuchProcess:
        return []
    except psutil.Error:
        children = []
    return children + [process]


def _terminate_windows_process_tree(
    logger: logging.Logger,
    label: str,
    *,
    expected_identity: PersistedProcessIdentity,
    terminate_timeout: float,
) -> bool:
    try:
        process, live_identity = _open_process_identity(expected_identity.pid)
    except psutil.NoSuchProcess:
        return True
    except (psutil.Error, OSError, ValueError):
        logger.debug("Failed to revalidate Windows supervisor for %s", label, exc_info=True)
        return False
    if process_identity_recycled(expected_identity, live_identity):
        return True
    if not process_identity_matches(expected_identity, live_identity):
        logger.warning("Refusing to signal %s because its supervisor marker changed", label)
        return False

    victims = _windows_process_tree(process)
    control_signal = getattr(signal, "CTRL_BREAK_EVENT", None)
    if control_signal is not None:
        try:
            process.send_signal(control_signal)
        except psutil.NoSuchProcess:
            # The captured descendants can still be alive after their leader exits.
            pass
        except (psutil.Error, OSError):
            # Windowless desktop children do not share a console with this
            # process. A failed console signal does not revoke the verified
            # identity: continue to the bounded, PID-reuse-safe psutil cleanup.
            logger.debug("Could not send a console signal to %s; checking its process tree", label, exc_info=True)

    _gone, alive = psutil.wait_procs(victims, timeout=terminate_timeout)
    if not alive:
        return True

    logger.warning("Escalating termination of %s after graceful timeout", label)
    for victim in reversed(alive):
        try:
            victim.terminate()
        except psutil.NoSuchProcess:
            continue
        except psutil.Error:
            logger.debug("Failed to terminate %s descendant pid=%s", label, victim.pid, exc_info=True)
    _gone, alive = psutil.wait_procs(alive, timeout=terminate_timeout)
    for victim in alive:
        try:
            victim.kill()
        except psutil.NoSuchProcess:
            continue
        except psutil.Error:
            logger.debug("Failed to kill %s descendant pid=%s", label, victim.pid, exc_info=True)
    _gone, alive = psutil.wait_procs(alive, timeout=terminate_timeout)
    if alive:
        logger.error("%s process tree survived forced termination", label)
        return False
    return True


def terminate_process_tree_by_pid(
    pid: int,
    logger: logging.Logger,
    label: str,
    *,
    expected_identity: PersistedProcessIdentity,
    terminate_timeout: float = DEFAULT_PROCESS_TERMINATE_TIMEOUT_SECONDS,
) -> bool:
    """Terminate an isolated process tree identified only by its root PID.

    This is intended for recovering persisted children after their original
    ``asyncio.subprocess.Process`` handle has been lost. Unlike
    :func:`signal_process_tree`, it never falls back to a direct signal when a
    POSIX target is in the service's own process group or its group cannot be
    verified.
    """
    if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0:
        return False
    if pid == os.getpid():
        logger.error("Refusing to terminate %s because pid=%s is the current process", label, pid)
        return False
    if not isinstance(expected_identity, PersistedProcessIdentity) or expected_identity.pid != pid:
        logger.error("Refusing to terminate %s pid=%s without a matching inspected identity", label, pid)
        return False

    try:
        process, live_identity = _open_process_identity(pid)
    except psutil.NoSuchProcess:
        return True
    except (psutil.Error, OSError, ValueError):
        logger.debug("Failed to inspect %s pid=%s before termination", label, pid, exc_info=True)
        return False
    if process_identity_recycled(expected_identity, live_identity):
        logger.warning("Refusing to terminate %s pid=%s because its process identity changed", label, pid)
        return True
    if not process_identity_matches(expected_identity, live_identity):
        logger.warning("Refusing to terminate %s pid=%s because its worker marker changed", label, pid)
        return False

    if os.name == "nt":
        return _terminate_windows_process_tree(
            logger,
            label,
            expected_identity=expected_identity,
            terminate_timeout=terminate_timeout,
        )

    try:
        pgid = os.getpgid(pid)
    except ProcessLookupError:
        return True
    except Exception:
        logger.debug("Failed to inspect %s process group pid=%s", label, pid, exc_info=True)
        return False
    if pgid != pid:
        logger.error(
            "Refusing to terminate %s pid=%s because its pgid=%s does not identify an isolated worker group",
            label,
            pid,
            pgid,
        )
        return False

    if not _safe_signal_known_process_group(
        pgid,
        pid,
        signal.SIGTERM,
        logger,
        label,
        expected_identity=expected_identity,
        require_leader=True,
    ):
        return False
    if _wait_for_process_group_exit(pgid, logger, label, timeout=terminate_timeout):
        return True

    logger.warning("Escalating termination of %s process group pgid=%s", label, pgid)
    if not _safe_signal_known_process_group(
        pgid,
        pid,
        KILL_SIGNAL,
        logger,
        label,
        expected_identity=expected_identity,
        require_leader=True,
    ):
        return False
    if _wait_for_process_group_exit(pgid, logger, label, timeout=terminate_timeout):
        return True
    logger.error("%s process group pgid=%s survived forced termination", label, pgid)
    return False


def orphaned_process_tree_presence(
    logger: logging.Logger,
    label: str,
    *,
    expected_identity: PersistedProcessIdentity,
) -> ProcessTreePresence:
    """Answer whether a recorded tree is still out there, WITHOUT signalling it.

    The read-only twin of ``reap_orphaned_process_tree``, asking the same question
    along the same two paths -- the leader by identity, then the group it leads -- for
    a caller with the opposite intent. The reap wants to stop a survivor; this wants
    to know whether starting a *second* copy of that work would collide with one, and
    a probe that killed the first copy to answer would be no answer at all.

    The three verdicts are not two. ``present`` and ``unknown`` both forbid a second
    copy, but for different reasons and with different remedies -- one is a running
    process, the other is a pass that could not look -- and only ``gone`` is proof
    that the recorded tree has stopped, which is what lets a caller both proceed and
    retire the record.
    """

    if not isinstance(expected_identity, PersistedProcessIdentity):
        # Nothing trustworthy was recorded, so there is no tree this can vouch for.
        return "gone"
    pid = expected_identity.pid
    liveness = probe_process_liveness(pid)
    if liveness == "unknown":
        return "unknown"
    if liveness == "alive":
        live_identity = inspect_process_identity(pid)
        if live_identity is None:
            # Occupied, but unreadable: cannot show it is ours, cannot show it is not.
            return "unknown"
        if process_identity_matches(expected_identity, live_identity):
            return "present"
        # A stranger holds the recycled number. Our leader has exited -- which, as the
        # reap's own comment says, only starts the question: the group may live on.

    if not process_group_exists(pid, logger, label):
        return "gone"
    status = process_group_identity_status(pid, expected_identity, logger, label)
    if status == "match":
        return "present"
    if status == "mismatch":
        # Members, every one of them another tree's: this pgid was recycled too.
        return "gone"
    return "unknown"


def reap_orphaned_process_tree(
    logger: logging.Logger,
    label: str,
    *,
    expected_identity: PersistedProcessIdentity,
) -> ProcessReapOutcome:
    """Kill whatever is left of a managed tree whose owner died, by identity.

    Recovery asks one question -- "may I stop tracking this process now?" -- and it
    has three answers, not two. ``gone`` and ``reaped`` retire the durable record
    that names the tree; ``unconfirmed`` must keep it, because the record is
    typically the only place the identity is written and discarding it makes the
    survivor unfindable by every later pass, not just this one.

    THE LEADER IS NOT THE TREE. A supervisor spawned with ``start_new_session``
    leads the group ``pgid == pid``, and on POSIX that group outlives it: if the
    supervisor is killed (an OOM kill, a crash in its own code) while the backup or
    migration under it keeps running, the pid is free while the work continues in a
    group nothing else names. So an empty pid only starts the question -- the group
    is checked next, and only a group with no members, or one whose members carry
    another tree's marker, means our tree is really gone.

    Identity throughout, never a bare pid or pgid: ``terminate_process_tree_by_pid``
    re-reads the leader's start time and ``AVIBE_PROCESS_IDENTITY`` marker, and
    ``terminate_process_group_by_pgid`` requires a surviving member to carry that
    marker, so a recycled number cannot make this signal a stranger's process.
    """

    if not isinstance(expected_identity, PersistedProcessIdentity):
        return "gone"
    pid = expected_identity.pid
    liveness = probe_process_liveness(pid)
    if liveness == "unknown":
        # The probe could not read the process table; that says nothing about the
        # process, so it cannot be the reason a handle on it is dropped.
        return "unconfirmed"
    if liveness == "alive":
        try:
            if terminate_process_tree_by_pid(
                pid,
                logger,
                label,
                expected_identity=expected_identity,
            ):
                return "reaped"
        except Exception:
            logger.exception("Unexpected error terminating %s pid=%s", label, pid)
        if probe_process_liveness(pid) != "gone":
            return "unconfirmed"
        # The leader went away without confirming; its children may not have.

    if not process_group_exists(pid, logger, label):
        return "gone"
    status = process_group_identity_status(pid, expected_identity, logger, label)
    if status == "mismatch":
        # Members, but every one of them carries some other tree's marker: this pgid
        # was recycled and ours has exited.
        return "gone"
    if status != "match":
        logger.warning(
            "Could not verify what still occupies %s process group pgid=%s; leaving it "
            "tracked for a later pass",
            label,
            pid,
        )
        return "unconfirmed"
    logger.warning(
        "Reaping %s process group pgid=%s after its leader exited",
        label,
        pid,
    )
    try:
        if terminate_process_group_by_pgid(
            pid,
            logger,
            label,
            expected_identity=expected_identity,
        ):
            return "reaped"
    except Exception:
        logger.exception("Unexpected error terminating %s process group pgid=%s", label, pid)
    return "unconfirmed"


def processes_carrying_marker(
    worker_fingerprint: str,
    *,
    marker_env: str = PROCESS_IDENTITY_ENV,
) -> list[psutil.Process]:
    """This user's live, inspectable processes whose ``marker_env`` hashes to ``worker_fingerprint``.

    Best effort by design. The marked tree is a same-user, non-setuid process
    this service spawned, so its ownership and environment are readable; a process
    whose ownership or environment cannot be read (another user's, a setuid
    ``login``, a non-dumpable Linux daemon) is not one of ours and is skipped. No
    birth-time heuristic is applied: create_time can drift, the marker cannot.
    """

    own_pid = os.getpid()
    own_uid = os.getuid() if hasattr(os, "getuid") else None
    own_user = None if own_uid is not None else psutil.Process(own_pid).username()
    found: list[psutil.Process] = []
    # The cached iteration skips, once, a pid psutil has flagged as reused, so
    # a rescan could miss that pid's new holder. Each scan starts fresh.
    psutil.process_iter.cache_clear()
    # psutil rejects an attribute its platform lacks, and Windows has no uids.
    for process in psutil.process_iter(["uids"] if own_uid is not None else ["username"]):
        if process.pid == own_pid:
            continue
        if own_uid is not None:
            uids = process.info.get("uids")
            if uids is None or {uids.real, uids.effective, uids.saved} != {own_uid}:
                continue
        elif process.info.get("username") != own_user:
            continue
        try:
            marker = process.environ().get(marker_env)
        except (psutil.Error, OSError):
            # Exited, or not inspectable and therefore not a tree this service spawned.
            continue
        try:
            fingerprint = fingerprint_process_marker(marker) if isinstance(marker, str) and marker else None
        except UnicodeError:
            continue
        if fingerprint is not None and hmac.compare_digest(fingerprint, worker_fingerprint):
            found.append(process)
    return found


def reap_marked_processes(
    logger: logging.Logger,
    label: str,
    *,
    worker_fingerprint: str,
    terminate_timeout: float = DEFAULT_PROCESS_TERMINATE_TIMEOUT_SECONDS,
    on_error: Callable[[BaseException], None] | None = None,
) -> ProcessReapOutcome:
    """Stop every process that inherited a managed tree's marker, wherever it is.

    Needs no pid: this finds a tree whose owner died before recording one, and
    members that left the leader's process group. Each victim is signalled through
    its retained ``psutil.Process``, which refuses a pid reused since the scan.
    ``unconfirmed`` only when a found process could not be confirmed stopped.
    """

    if not is_valid_worker_fingerprint(worker_fingerprint):
        return "gone"
    try:
        victims = processes_carrying_marker(worker_fingerprint)
        if not victims:
            return "gone"
        logger.warning("Reaping %d %s process(es) found by their identity marker", len(victims), label)
        for victim in victims:
            try:
                victim.terminate()
            except psutil.NoSuchProcess:
                continue
        _gone, alive = psutil.wait_procs(victims, timeout=terminate_timeout)
        for victim in alive:
            try:
                victim.kill()
            except psutil.NoSuchProcess:
                continue
        _gone, alive = psutil.wait_procs(alive, timeout=terminate_timeout)
        if not alive and not processes_carrying_marker(worker_fingerprint):
            return "reaped"
    except Exception as exc:
        if on_error is not None:
            on_error(exc)
        logger.warning("Could not confirm every %s process carrying its marker exited", label, exc_info=True)
    return "unconfirmed"


async def terminate_process_tree(
    process: Any,
    logger: logging.Logger,
    label: str,
    *,
    terminate_timeout: float = DEFAULT_PROCESS_TERMINATE_TIMEOUT_SECONDS,
) -> None:
    """Terminate a managed subprocess without signaling the service group."""
    if getattr(process, "returncode", None) is not None:
        return

    signal_process_tree(process, signal.SIGTERM, logger, label)
    try:
        await asyncio.wait_for(process.wait(), timeout=terminate_timeout)
        return
    except asyncio.TimeoutError:
        pass

    signal_process_tree(process, KILL_SIGNAL, logger, label)
    try:
        await process.wait()
    except ProcessLookupError:
        return


async def terminate_and_communicate(
    process: Any,
    logger: logging.Logger,
    label: str,
    *,
    terminate_timeout: float = DEFAULT_PROCESS_TERMINATE_TIMEOUT_SECONDS,
) -> tuple[bytes, bytes]:
    """Terminate a process tree and drain stdout/stderr."""
    if getattr(process, "returncode", None) is None:
        signal_process_tree(process, signal.SIGTERM, logger, label)
    try:
        return await asyncio.wait_for(process.communicate(), timeout=terminate_timeout)
    except asyncio.TimeoutError:
        signal_process_tree(process, KILL_SIGNAL, logger, label)
        return await process.communicate()