Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

68.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Kills Unsloth children when the parent dies abnormally (Job Object, PDEATHSIG, macOS disk record
swept at startup); best-effort, never raises; stdlib only."""

from __future__ import annotations

import os
import signal
import sys
import contextvars
import functools
import threading
import time
from typing import Callable, Optional

_PR_SET_PDEATHSIG = 1
_JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
_JobObjectExtendedLimitInformation = 9

# killpg(1, sig) is kill(-1, sig): signals everything the user owns. Never record or signal < 2.
_LOWEST_SIGNALABLE_PID = 2


def is_signalable_pid(pid: object) -> bool:
    """Excludes bool, since True would read as pid 1; public so every signalling site shares this floor."""
    return isinstance(pid, int) and not isinstance(pid, bool) and pid >= _LOWEST_SIGNALABLE_PID


# Internal alias for is_signalable_pid, kept so this module's call sites and tests need no change.
_signalable = is_signalable_pid


_lock = threading.Lock()
_spawner_lock = threading.Lock()
_spawner: "Optional[_Spawner]" = None
_initialized = False
_win_job_handle: Optional[int] = None  # retained for the interpreter's lifetime
_tracked_pids: "dict[int, Optional[str]]" = {}  # pid -> identity, reaped by terminate_all
# The group outlives an exited leader and is the only handle on its children.
_tracked_pgids: "dict[int, int]" = {}
# Serialises edit-then-write so concurrent adopts do not drop a pid.
_record_lock = threading.Lock()


# A silent failure leaks every child on a crash, so record and log it.
_win_job_status: "tuple[bool, str]" = (False, "not attempted")


def _last_error(ctypes_module) -> int:
    # get_last_error is Windows-only; a POSIX probe must not raise here.
    getter = getattr(ctypes_module, "get_last_error", None)
    try:
        return int(getter()) if getter else 0
    except Exception:
        return 0


def _record_job_status(
    ok: bool,
    detail: str,
    last_error: int = 0,
) -> None:
    global _win_job_status
    if last_error:
        detail = f"{detail} (WinError {last_error})"
    _win_job_status = (ok, detail)
    try:
        import logging
        logger = logging.getLogger(__name__)
        if ok:
            logger.info("Child-process cleanup on abnormal exit: %s", detail)
        else:
            logger.warning(
                "Child-process cleanup on abnormal exit is NOT guaranteed: %s. Children may "
                "survive a crash or a force quit; the startup sweep reaps them on the next "
                "launch.",
                detail,
            )
    except Exception:
        pass


def windows_job_status() -> "tuple[bool, str]":
    """(in_force, detail) for the Windows kill-on-close job."""
    return _win_job_status


def _is_linux() -> bool:
    return sys.platform.startswith("linux")


def _is_windows() -> bool:
    return sys.platform == "win32"


# ── Parent setup ──


def initialize_parent_lifetime() -> None:
    """Idempotent and never raises; on Windows it holds the Job Object, while POSIX installs nothing."""
    global _initialized
    with _lock:
        if _initialized:
            return
        _initialized = True
        if _is_windows():
            _install_windows_job()
        elif _is_linux():
            if _pdeathsig_available():
                _record_job_status(True, "PR_SET_PDEATHSIG per child")
            else:
                _record_job_status(False, "prctl is unavailable here (seccomp or container policy)")
        else:
            # macOS: no pdeathsig, no job object; reap_recorded_children covers it.
            _record_job_status(False, "no kernel-level parent-death signal on this platform")


def _pdeathsig_available() -> bool:
    """A read-only PR_GET proves nothing under seccomp, so the probe does a no-op PR_SET instead."""
    _PR_GET_PDEATHSIG, _PR_SET_PDEATHSIG = 2, 1
    try:
        import ctypes

        libc = ctypes.CDLL("libc.so.6", use_errno = True)
        current = ctypes.c_int(0)
        if libc.prctl(_PR_GET_PDEATHSIG, ctypes.byref(current), 0, 0, 0) != 0:
            return False
        return libc.prctl(_PR_SET_PDEATHSIG, current.value, 0, 0, 0) == 0
    except Exception:
        return False


def _win_signatures(kernel32) -> None:
    # Explicit HANDLE signatures: c_int defaults truncate handles on Win64.
    import ctypes
    from ctypes import wintypes

    H, BOOL, DWORD = wintypes.HANDLE, wintypes.BOOL, wintypes.DWORD
    kernel32.CreateJobObjectW.argtypes = [ctypes.c_void_p, ctypes.c_wchar_p]
    kernel32.CreateJobObjectW.restype = H
    kernel32.SetInformationJobObject.argtypes = [H, ctypes.c_int, ctypes.c_void_p, DWORD]
    kernel32.SetInformationJobObject.restype = BOOL
    kernel32.AssignProcessToJobObject.argtypes = [H, H]
    kernel32.AssignProcessToJobObject.restype = BOOL
    kernel32.GetCurrentProcess.argtypes = []
    kernel32.GetCurrentProcess.restype = H
    kernel32.CloseHandle.argtypes = [H]
    kernel32.CloseHandle.restype = BOOL


def _install_windows_job() -> None:
    global _win_job_handle
    try:
        import ctypes
        from ctypes import wintypes

        kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
        _win_signatures(kernel32)

        class _BASIC(ctypes.Structure):
            _fields_ = [
                ("PerProcessUserTimeLimit", ctypes.c_int64),
                ("PerJobUserTimeLimit", ctypes.c_int64),
                ("LimitFlags", wintypes.DWORD),
                ("MinimumWorkingSetSize", ctypes.c_size_t),
                ("MaximumWorkingSetSize", ctypes.c_size_t),
                ("ActiveProcessLimit", wintypes.DWORD),
                ("Affinity", ctypes.c_size_t),
                ("PriorityClass", wintypes.DWORD),
                ("SchedulingClass", wintypes.DWORD),
            ]

        class _IO(ctypes.Structure):
            _fields_ = [
                (n, ctypes.c_uint64)
                for n in (
                    "ReadOperationCount",
                    "WriteOperationCount",
                    "OtherOperationCount",
                    "ReadTransferCount",
                    "WriteTransferCount",
                    "OtherTransferCount",
                )
            ]

        class _EXT(ctypes.Structure):
            _fields_ = [
                ("BasicLimitInformation", _BASIC),
                ("IoInfo", _IO),
                ("ProcessMemoryLimit", ctypes.c_size_t),
                ("JobMemoryLimit", ctypes.c_size_t),
                ("PeakProcessMemoryUsed", ctypes.c_size_t),
                ("PeakJobMemoryUsed", ctypes.c_size_t),
            ]

        job = kernel32.CreateJobObjectW(None, None)
        if not job:
            _record_job_status(False, "CreateJobObjectW failed", _last_error(ctypes))
            return
        info = _EXT()
        info.BasicLimitInformation.LimitFlags = _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
        if not kernel32.SetInformationJobObject(
            job, _JobObjectExtendedLimitInformation, ctypes.byref(info), ctypes.sizeof(info)
        ):
            _record_job_status(False, "SetInformationJobObject failed", _last_error(ctypes))
            kernel32.CloseHandle(job)
            return
        # May fail inside an incompatible host job (pre-Win8); degrade, do not block startup.
        if not kernel32.AssignProcessToJobObject(job, kernel32.GetCurrentProcess()):
            _record_job_status(False, "AssignProcessToJobObject failed", _last_error(ctypes))
            kernel32.CloseHandle(job)
            return
        _win_job_handle = job
        _record_job_status(True, "kill-on-close job installed")
    except Exception as error:
        _record_job_status(False, f"{type(error).__name__}: {error}")


# ── Child binding ──


def _pdeathsig_preexec(owner_pid: Optional[int] = None) -> None:
    # Runs pre-exec in the child. Compare getppid to owner_pid, not 1: a parent can legitimately
    # be pid 1 (container entrypoint), and subreapers are not pid 1.
    try:
        import ctypes

        ctypes.CDLL("libc.so.6", use_errno = True).prctl(_PR_SET_PDEATHSIG, signal.SIGTERM)
        parent_pid = os.getppid()
        orphaned = parent_pid != owner_pid if owner_pid is not None else parent_pid == 1
        if orphaned:
            os._exit(1)
    except Exception:
        pass


def bind_current_process_to_parent_lifetime() -> None:
    """Bind the CURRENT process to its parent's death (Linux). For multiprocessing
    children, which cannot take a preexec_fn, so the parent cannot set
    PR_SET_PDEATHSIG for them -- the child must do it itself at startup."""
    if not _is_linux():
        return
    parent = None
    try:
        import multiprocessing
        parent = multiprocessing.parent_process()
    except Exception:
        pass
    if parent is None:
        # Only arm PDEATHSIG: a getppid() == 1 test would kill a parent-is-pid-1 process.
        _pdeathsig_preexec(os.getppid())
        return
    # Orphan check uses the creator's sentinel, not a pid compare: under forkserver the kernel
    # parent is the fork server.
    _pdeathsig_preexec(os.getppid())
    try:
        if not parent.is_alive():
            os._exit(1)
    except Exception:
        pass


def allow_child_processes() -> None:
    """Grandchildren inherit the cleared flag and hang worker exit unless they pass daemon = True."""
    try:
        from multiprocessing import process as multiprocessing_process

        # This config is child-local; the parent's Process handle stays daemonic.
        config = getattr(multiprocessing_process.current_process(), "_config", None)
        if isinstance(config, dict):
            config["daemon"] = False
    except Exception:
        pass


def compose_preexec(
    existing: Optional[Callable[[], None]], owner_pid: Optional[int] = None
) -> Optional[Callable[[], None]]:
    """Run the PDEATHSIG hook then any caller-supplied preexec (Linux only)."""
    if not _is_linux():
        return existing

    def _composed() -> None:
        _pdeathsig_preexec(owner_pid)
        if existing is not None:
            existing()

    return _composed


_fork_reset_installed = False


def _reset_after_fork() -> None:
    """Fork children inherit locks in any state and a _spawner with no thread; start clean."""
    global _spawner, _spawner_lock, _record_lock, _owner_identity, _shutdown_latch
    _spawner_lock = threading.Lock()
    # A different pid here.
    _owner_identity = None
    _tracked_pids.clear()
    _tracked_pgids.clear()
    # Recreate after fork: a lock held by another thread at fork time is never released.
    _record_lock = threading.Lock()
    # Event has an internal lock too; rebuild it keeping its latched state, or a cleared latch would read as permission to spawn.
    _was_latched = _shutdown_latch.is_set()
    _shutdown_latch = threading.Event()
    if _was_latched:
        _shutdown_latch.set()
    _spawner = None


class _UncachedReformatters(threading.local):
    """PyAV's per-thread scaler cache while a fork is in flight: reads find nothing, writes are dropped."""

    def __setattr__(self, name: str, value: object) -> None:
        if name != "reformatter":
            super().__setattr__(name, value)


_forks_in_flight = 0


def _suspend_native_caches() -> None:
    """Frees PyAV's FFmpeg scaler caches before a fork; a child would hang waiting on threads it lacks."""
    global _forks_in_flight
    _forks_in_flight += 1
    try:
        frame = sys.modules.get("av.video.frame")
        cache = getattr(frame, "_thread_local", None)
        if isinstance(cache, threading.local) and not isinstance(cache, _UncachedReformatters):
            frame._thread_local = _UncachedReformatters()
    except Exception:  # noqa: BLE001 - a fork hook must not get in the fork's way
        pass


def _resume_native_caches() -> None:
    try:
        frame = sys.modules.get("av.video.frame")
        if isinstance(getattr(frame, "_thread_local", None), _UncachedReformatters):
            frame._thread_local = threading.local()
    except Exception:  # noqa: BLE001
        pass


def _resume_native_caches_in_parent() -> None:
    global _forks_in_flight
    _forks_in_flight = max(0, _forks_in_flight - 1)
    # Another thread's fork may still be between its own hooks.
    if _forks_in_flight == 0:
        _resume_native_caches()


def _resume_native_caches_in_child() -> None:
    global _forks_in_flight
    _forks_in_flight = 0
    _resume_native_caches()


# At import, not lazily: run_server imports this module before anything spawns, and every fork needs it, including
# the ones that bring their own preexec_fn or call os.fork directly.
if hasattr(os, "register_at_fork"):
    os.register_at_fork(
        before = _suspend_native_caches,
        after_in_parent = _resume_native_caches_in_parent,
        after_in_child = _resume_native_caches_in_child,
    )


def _adopt_fork_reset() -> None:
    """Register the child-side reset once, lazily. Best-effort like the rest of
    this module: os.register_at_fork is POSIX-only and absent on Windows."""
    global _fork_reset_installed
    if _fork_reset_installed:
        return
    _fork_reset_installed = True
    try:
        os.register_at_fork(after_in_child = _reset_after_fork)
    except (AttributeError, RuntimeError):
        pass


def spawn_on_lifetime_thread(spawn: Callable[[], object]) -> object:
    """PR_SET_PDEATHSIG fires when the forking thread exits, so spawn from a process-lifetime thread."""
    if not _is_linux():
        return spawn()
    global _spawner
    _adopt_fork_reset()
    with _spawner_lock:
        if _spawner is None or not _spawner.usable():
            candidate = _Spawner()
            _spawner = candidate if candidate.usable() else None
        spawner = _spawner
    # No helper thread (tests replace threading.Thread): spawn inline.
    return spawn() if spawner is None else spawner.run(spawn)


class _Spawner:
    """One daemon thread that forks on behalf of any caller. Daemon is correct:
    the process dies at interpreter exit, which is when children should die."""

    def __init__(self) -> None:
        import queue

        self._jobs: "queue.Queue" = queue.Queue()
        self._ready = threading.Event()
        self._thread = None
        try:
            self._thread = threading.Thread(
                target = self._run, name = "unsloth-child-spawner", daemon = True
            )
            self._thread.start()
        except Exception:  # noqa: BLE001 - fall back to an inline spawn
            self._thread = None

    def usable(self) -> bool:
        """True only once the helper has actually begun running."""
        if self._thread is None:
            return False
        return self._ready.wait(5.0) and bool(getattr(self._thread, "is_alive", bool)())

    def _run(self) -> None:
        self._ready.set()
        while True:
            spawn, box, done = self._jobs.get()
            try:
                box.append((True, spawn()))
            except BaseException as exc:  # noqa: BLE001 - re-raised in the caller
                box.append((False, exc))
            finally:
                done.set()

    def run(self, spawn: Callable[[], object]) -> object:
        box: list = []
        done = threading.Event()
        # The spawner thread has no context of its own; run the spawn in the caller's.
        self._jobs.put((functools.partial(contextvars.copy_context().run, spawn), box, done))
        done.wait()
        ok, value = box[0]
        if not ok:
            raise value
        return value


def child_popen_kwargs(preexec_fn: Optional[Callable[[], None]] = None) -> dict:
    """The returned preexec_fn sets PDEATHSIG on Linux; Windows relies on the inherited Job Object."""
    if _is_linux():
        # getpid() in the spawner lets the child tell reparenting from a pid-1 parent.
        return {"preexec_fn": compose_preexec(preexec_fn, os.getpid())}
    return {}


def _recorded_identity(value: object) -> "Optional[str]":
    """An identity read back from a record, or None when it is not one."""
    return value if isinstance(value, str) and value else None


def _same_identity(recorded: str, current: str) -> bool:
    """Old Linux records hold starttime:comm; compare only the start time there, else the whole string."""
    # A malformed record must not raise, or one bad file aborts the whole sweep.
    if not isinstance(recorded, str) or not isinstance(current, str):
        return False
    if _is_linux():
        return recorded.split(":", 1)[0] == current.split(":", 1)[0]
    if sys.platform == "darwin":
        # lstart alone, as on Linux: a framework python re-execs into Python.app, changing comm.
        recorded_start, current_start = recorded.split()[:5], current.split()[:5]
        if len(recorded_start) == 5 and len(current_start) == 5:
            return recorded_start == current_start
    return recorded == current


def _pid_identity(pid: int) -> Optional[str]:
    # Start time pins identity so a reused pid is never signalled later.
    if _is_linux():
        try:
            with open(f"/proc/{pid}/stat", encoding = "utf-8") as fh:
                stat = fh.read()
            # Start time only: comm is mutable, so a renamed child would be dropped unsignalled.
            return stat[stat.rfind(")") + 2 :].split()[19]
        except Exception:
            return None
    if _is_windows():
        # Creation time, so a recycled pid is never mistaken for the recorded child; same purpose as starttime on Linux.
        try:
            import ctypes
            from ctypes import wintypes

            PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
            kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
            kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
            kernel32.OpenProcess.restype = wintypes.HANDLE
            kernel32.GetProcessTimes.argtypes = [wintypes.HANDLE] + [
                ctypes.POINTER(wintypes.FILETIME)
            ] * 4
            kernel32.GetProcessTimes.restype = wintypes.BOOL
            kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
            kernel32.CloseHandle.restype = wintypes.BOOL
            handle = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
            if not handle:
                return None
            try:
                created = wintypes.FILETIME()
                other = [wintypes.FILETIME() for _ in range(3)]
                if not kernel32.GetProcessTimes(
                    handle, ctypes.byref(created), *[ctypes.byref(x) for x in other]
                ):
                    return None
                return f"{created.dwHighDateTime}:{created.dwLowDateTime}"
            finally:
                kernel32.CloseHandle(handle)
        except Exception:
            return None
    if sys.platform == "darwin":
        try:
            import subprocess

            # TZ pinned: lstart is formatted in local time.
            out = subprocess.run(
                ["ps", "-o", "lstart=,comm=", "-p", str(pid)],
                capture_output = True,
                text = True,
                encoding = "utf-8",
                errors = "replace",
                timeout = 5,
                env = {**os.environ, "TZ": "UTC"},
            )
            line = (out.stdout or "").strip()
            return line or None
        except Exception:
            return None
    return None


def forget_pid(pid: Optional[int]) -> None:
    """Stop tracking a reaped child so terminate_all never signals a recycled pid; kept while its group has members, since the shim can exit first."""
    if not pid:
        return
    with _record_lock:
        if pid not in _tracked_pids and pid not in _tracked_pgids:
            return  # never recorded, so there is nothing to rewrite or check
        if _group_has_members(_tracked_pgids.get(pid)):
            return
        _tracked_pids.pop(pid, None)
        _tracked_pgids.pop(pid, None)
        _write_breadcrumb()


def _group_has_members(pgid: object) -> bool:
    """killpg(pgid, 0) also succeeds for zombie-only groups, which would then keep their record forever."""
    # killpg(1, 0) always succeeds, so without the floor a poisoned pgid retries forever.
    if not _signalable(pgid) or _is_windows() or not hasattr(os, "killpg"):
        return False
    try:
        os.killpg(pgid, 0)
    except Exception:
        return False
    # Check the leader first: enumerating a group scans every process (about 62ms at 6000).
    if _pid_alive(pgid) and not _pid_is_zombie(pgid):
        return True
    members = _group_member_pids(pgid)
    if members is None:
        return True
    return any(not _pid_is_zombie(pid) for pid in members)


def _windows_creation_time(identity: "Optional[str]") -> "Optional[int]":
    """Parses the high:low identity to one 64-bit FILETIME, so creation times can be ordered."""
    if not isinstance(identity, str):
        return None
    parts = identity.split(":")
    if len(parts) != 2:
        return None
    try:
        return (int(parts[0]) << 32) | int(parts[1])
    except ValueError:
        return None


def _windows_identity_of_handle(kernel32, handle) -> "Optional[str]":
    """Read through an open handle, which pins the process; None means not proven, never a match."""
    try:
        import ctypes
        from ctypes import wintypes

        kernel32.GetProcessTimes.argtypes = [wintypes.HANDLE] + [
            ctypes.POINTER(wintypes.FILETIME)
        ] * 4
        kernel32.GetProcessTimes.restype = wintypes.BOOL
        created = wintypes.FILETIME()
        other = [wintypes.FILETIME() for _ in range(3)]
        if not kernel32.GetProcessTimes(
            handle, ctypes.byref(created), *[ctypes.byref(x) for x in other]
        ):
            return None
        return f"{created.dwHighDateTime}:{created.dwLowDateTime}"
    except Exception:  # noqa: BLE001 -- unreadable is "not proven", handled by the caller
        return None


def _windows_terminate_through_a_handle(pid: int, identity: "Optional[str]") -> "Optional[bool]":
    """Kills only via a handle proven to be the process (taskkill and os.kill re-resolve the pid); None means fall back, never kill unproven."""
    if identity is None:
        return None
    try:
        import ctypes
        from ctypes import wintypes

        PROCESS_TERMINATE = 0x0001
        PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
        kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
        kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
        kernel32.OpenProcess.restype = wintypes.HANDLE
        kernel32.TerminateProcess.argtypes = [wintypes.HANDLE, wintypes.UINT]
        kernel32.TerminateProcess.restype = wintypes.BOOL
        kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
        kernel32.CloseHandle.restype = wintypes.BOOL
        handle = kernel32.OpenProcess(
            PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION, False, pid
        )
        if not handle:
            # Gone, protected, or denied: cannot answer, not a wrong process.
            return None
        try:
            opened = _windows_identity_of_handle(kernel32, handle)
            if opened is None or opened != identity:
                # Either the pid now belongs to something else, or it cannot be shown to be the recorded process; neither is a licence to kill.
                return False
            return bool(kernel32.TerminateProcess(handle, 1))
        finally:
            kernel32.CloseHandle(handle)
    except Exception:  # noqa: BLE001 -- no ctypes, no kernel32: the caller falls back
        return None


def _windows_filetime_now() -> "Optional[int]":
    """Wall clock as a FILETIME; a process created after a pre-snapshot reading was not in the snapshot."""
    if not _is_windows():
        return None
    try:
        import ctypes
        from ctypes import wintypes

        kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
        kernel32.GetSystemTimeAsFileTime.argtypes = [ctypes.POINTER(wintypes.FILETIME)]
        kernel32.GetSystemTimeAsFileTime.restype = None
        stamp = wintypes.FILETIME()
        kernel32.GetSystemTimeAsFileTime(ctypes.byref(stamp))
        return (int(stamp.dwHighDateTime) << 32) | int(stamp.dwLowDateTime)
    except Exception:  # noqa: BLE001 -- cannot tell; the walk keeps its old behaviour
        return None


def _windows_child_pid_map() -> "Optional[dict[int, list[int]]]":
    """Uses ctypes and a Toolhelp snapshot, not psutil or a helper, since teardown must not spawn one."""
    try:
        import ctypes
        from ctypes import wintypes

        TH32CS_SNAPPROCESS = 0x0000_0002
        MAX_PATH = 260

        class PROCESSENTRY32W(ctypes.Structure):
            _fields_ = [
                ("dwSize", wintypes.DWORD),
                ("cntUsage", wintypes.DWORD),
                ("th32ProcessID", wintypes.DWORD),
                ("th32DefaultHeapID", ctypes.c_size_t),
                ("th32ModuleID", wintypes.DWORD),
                ("cntThreads", wintypes.DWORD),
                ("th32ParentProcessID", wintypes.DWORD),
                ("pcPriClassBase", ctypes.c_long),
                ("dwFlags", wintypes.DWORD),
                ("szExeFile", wintypes.WCHAR * MAX_PATH),
            ]

        kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
        kernel32.CreateToolhelp32Snapshot.argtypes = [wintypes.DWORD, wintypes.DWORD]
        kernel32.CreateToolhelp32Snapshot.restype = wintypes.HANDLE
        kernel32.Process32FirstW.argtypes = [wintypes.HANDLE, ctypes.POINTER(PROCESSENTRY32W)]
        kernel32.Process32FirstW.restype = wintypes.BOOL
        kernel32.Process32NextW.argtypes = [wintypes.HANDLE, ctypes.POINTER(PROCESSENTRY32W)]
        kernel32.Process32NextW.restype = wintypes.BOOL
        kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
        kernel32.CloseHandle.restype = wintypes.BOOL

        snapshot = kernel32.CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0)
        # INVALID_HANDLE_VALUE (-1) arrives as a large unsigned value, not falsy.
        if not snapshot or snapshot == ctypes.c_void_p(-1).value:
            return None
        try:
            entry = PROCESSENTRY32W()
            entry.dwSize = ctypes.sizeof(PROCESSENTRY32W)
            if not kernel32.Process32FirstW(snapshot, ctypes.byref(entry)):
                # An empty walk is a failed read, not a machine with no processes.
                return None
            ERROR_NO_MORE_FILES = 18
            table: "dict[int, list[int]]" = {}
            while True:
                child = int(entry.th32ProcessID)
                parent = int(entry.th32ParentProcessID)
                if child:
                    table.setdefault(parent, []).append(child)
                if not kernel32.Process32NextW(snapshot, ctypes.byref(entry)):
                    # FALSE also means a mid-walk failure; only ERROR_NO_MORE_FILES ends the list, or unreached workers lose their record and pidfile.
                    if ctypes.get_last_error() != ERROR_NO_MORE_FILES:
                        return None
                    break
            return table
        finally:
            kernel32.CloseHandle(snapshot)
    except Exception:
        return None


def _child_pid_map() -> "Optional[dict[int, list[int]]]":
    """Parent pid -> its children, or None when the table cannot be read."""
    if _is_linux():
        try:
            table: "dict[int, list[int]]" = {}
            for entry in os.listdir("/proc"):
                if not entry.isdigit():
                    continue
                try:
                    with open(f"/proc/{entry}/stat", encoding = "utf-8") as fh:
                        stat = fh.read()
                except OSError:
                    continue
                # After the comm field: state, ppid, pgrp, ...
                tail = stat[stat.rfind(")") + 2 :].split()
                if len(tail) > 1:
                    table.setdefault(int(tail[1]), []).append(int(entry))
            return table
        except Exception:
            return None
    if sys.platform == "darwin":
        try:
            import subprocess

            out = subprocess.run(
                ["ps", "-A", "-o", "pid=,ppid="],
                capture_output = True,
                text = True,
                encoding = "utf-8",
                errors = "replace",
                timeout = 5,
            )
            if out.returncode != 0:
                return None
            table = {}
            for line in (out.stdout or "").splitlines():
                parts = line.split()
                if len(parts) != 2:
                    continue
                try:
                    child, parent = int(parts[0]), int(parts[1])
                except ValueError:
                    continue
                table.setdefault(parent, []).append(child)
            return table
        except Exception:
            return None
    if _is_windows():
        return _windows_child_pid_map()
    return None


def collect_descendants_known(
    pid: "Optional[int]",
) -> "tuple[list[tuple[int, Optional[str]]], bool]":
    """known is False when the walk failed; an unreadable walk must never be read as no children."""
    if not pid:
        # Not a question about a process, so there is nothing indeterminate about it.
        return [], True
    if not is_signalable_pid(pid):
        # Below the floor, pid 1 is never walked: every host process descends from init, so a tree
        # kill would reach the whole machine (a test stand-in at pid 1 took a user's session down).
        return [], True
    # Windows keeps a creating pid after its parent exits and the number is reused, so each
    # candidate is ordered against its parent's current creation time (a child cannot predate it) at
    # every depth; with no readable floor nothing is claimed and unreadable candidates are skipped.
    if _is_windows():
        return _windows_collect_descendants_known(pid)
    # POSIX: the kernel rewrites parent links on reparent, so no floor is needed.
    table = _child_pid_map()
    if not table:
        return [], False
    found: "list[tuple[int, Optional[str]]]" = []
    seen = {pid}
    queue = list(table.get(pid, ()))
    while queue:
        child = queue.pop(0)
        if child in seen:
            continue
        seen.add(child)
        found.append((child, _pid_identity(child)))
        queue.extend(table.get(child, ()))
    return found, True


def collect_descendants(pid: "Optional[int]") -> "list[tuple[int, Optional[str]]]":
    """Call before signalling the parent, since its children are reparented on exit and lose the link."""
    return collect_descendants_known(pid)[0]


def _windows_collect_descendants(pid: int) -> "list[tuple[int, Optional[str]]]":
    """The list alone. See `_windows_collect_descendants_known`."""
    return _windows_collect_descendants_known(pid)[0]


def _windows_collect_descendants_known(
    pid: int, identity: "Optional[str]" = None
) -> "tuple[list[tuple[int, Optional[str]]], bool]":
    """Orders each child against its parent creation time (Windows keeps stale creator pids); unreadable candidates are skipped and make the walk incomplete."""
    if identity is not None and not _provably_the_same(pid, identity):
        return [], False
    root_floor = _windows_creation_time(identity if identity is not None else _pid_identity(pid))
    if root_floor is None:
        # No floor means no ancestry proof: indeterminate, not empty.
        return [], False
    # Floor read before the snapshot: anything created after it is unprovable, so the walk
    # reports incomplete and later rounds re-snapshot.
    snapshot_floor = _windows_filetime_now()
    table = _child_pid_map()
    # Ceiling read after the snapshot: a process created later cannot be one the snapshot listed.
    # Taking it first would wrongly reject genuine children born in the gap.
    snapshot_ceiling = _windows_filetime_now()
    if not table:
        return [], False
    found: "list[tuple[int, Optional[str]]]" = []
    complete = True
    seen = {pid}
    # (candidate pid, creation time of the parent that listed it)
    queue: "list[tuple[int, int]]" = [(child, root_floor) for child in table.get(pid, ())]
    while queue:
        child, parent_created = queue.pop(0)
        if child in seen:
            continue
        seen.add(child)
        identity = _pid_identity(child)
        created = _windows_creation_time(identity)
        if created is None:
            # Unreadable but alive means the tree is not fully enumerated.
            if _pid_alive(child) and not _pid_is_zombie(child):
                complete = False
            continue
        if created < parent_created:
            # Predates its listed creator: recycled number, rejected; walk stays complete.
            continue
        if snapshot_ceiling is not None and created > snapshot_ceiling:
            # Did not exist at snapshot time, so the listed process exited and its pid was reused: skip it, the walk stays complete, and its subtree is not ours.
            continue
        if snapshot_floor is not None and created > snapshot_floor:
            # Created during the snapshot: unprovable, so skip and mark the walk incomplete.
            complete = False
            continue
        found.append((child, identity))
        queue.extend((grandchild, created) for grandchild in table.get(child, ()))
    return found, complete


# Kills are asynchronous on both platforms, so wait briefly before calling a pid a survivor;
# false survivors keep records and cause reap sweeps on every later launch.
_KILL_SETTLE_SECONDS = 0.5
_KILL_SETTLE_POLL_SECONDS = 0.01

# Two consecutive gone reads required: _pid_alive fails open on Windows.
# Doubling backoff: off Linux/Windows each probe forks ps.
_KILL_SETTLE_CONFIRMATIONS = 2
_KILL_SETTLE_POLL_CEILING_SECONDS = 0.1


def _survivors_after_settling(
    candidates: "list[tuple[int, Optional[str]]]",
    still_a_survivor: "Callable[[int, Optional[str]], bool]",
    grace: float = _KILL_SETTLE_SECONDS,
) -> "list[tuple[int, Optional[str]]]":
    """A pid drops out after _KILL_SETTLE_CONFIRMATIONS consecutive gone reads; a stuck pid stays reported and the callers order (deepest first) is kept."""
    if not candidates:
        return []
    agreed: "dict[int, int]" = {}

    def _pass() -> "list[tuple[int, Optional[str]]]":
        still: "list[tuple[int, Optional[str]]]" = []
        for pid, identity in candidates:
            if agreed.get(pid, 0) >= _KILL_SETTLE_CONFIRMATIONS:
                continue
            if still_a_survivor(pid, identity):
                agreed[pid] = 0
                still.append((pid, identity))
            else:
                agreed[pid] = agreed.get(pid, 0) + 1
                if agreed[pid] < _KILL_SETTLE_CONFIRMATIONS:
                    still.append((pid, identity))
        return still

    remaining = _pass()
    deadline = time.monotonic() + max(0.0, grace)
    wait = _KILL_SETTLE_POLL_SECONDS
    while remaining and time.monotonic() < deadline:
        time.sleep(wait)
        wait = min(wait * 2, _KILL_SETTLE_POLL_CEILING_SECONDS)
        remaining = _pass()
    return remaining


def confirm_pid_exited(pid: "Optional[int]", grace: float = _KILL_SETTLE_SECONDS) -> bool:
    """Waits out the settle grace; False when unknown, since True authorises dropping the last handles."""
    if not _signalable(pid):
        return False
    return not _survivors_after_settling(
        [(int(pid), None)], lambda candidate, _identity: pid_is_running(candidate), grace
    )


def _settle_after_the_kill(
    pid: int,
    group_leader: bool,
    grace: float = _KILL_SETTLE_SECONDS,
) -> None:
    """Waits briefly after SIGKILL, since an immediate liveness read still sees the child running."""

    def _still_there(candidate: int, _identity: "Optional[str]") -> bool:
        # A leader can be gone while its session is not.
        if group_leader and _group_has_members(candidate):
            return True
        return _pid_alive(candidate) and not _pid_is_zombie(candidate)

    _survivors_after_settling([(pid, None)], _still_there, grace)


def terminate_descendants(
    collected: "list[tuple[int, Optional[str]]]", timeout: float = 5.0
) -> "list[tuple[int, Optional[str]]]":
    """Returns survivors with the collected identity; a survivor must never read as an empty list."""
    if not collected:
        return []
    if _is_windows():
        return _windows_terminate_collected(collected)
    live: "list[tuple[int, Optional[str]]]" = []
    for pid, identity in collected:
        if not _signalable(pid) or not _still_the_same(pid, identity):
            continue
        try:
            os.kill(pid, signal.SIGTERM)
        except OSError:
            continue
        live.append((pid, identity))
    deadline = time.monotonic() + max(0.0, timeout)
    while live:
        live = [item for item in live if _pid_alive(item[0]) and not _pid_is_zombie(item[0])]
        if not live or time.monotonic() >= deadline:
            break
        time.sleep(0.05)
    for pid, identity in live:
        if not _still_the_same(pid, identity):
            continue
        try:
            os.kill(pid, signal.SIGKILL)
        except OSError:
            pass
    # Re-read after a grace: SIGKILL can be outlived (D state) and dying processes linger briefly.
    return _survivors_after_settling(
        live,
        lambda pid, identity: (
            _pid_alive(pid) and not _pid_is_zombie(pid) and _still_the_same(pid, identity)
        ),
    )


# Capture-before-kill recursion depth; exhausting it returns None (unresolved), never drops.
_SUBTREE_CAPTURE_DEPTH = 8


def _windows_kill_below(
    anchor: int,
    attempted: "list[tuple[int, Optional[str]]]",
    unresolved: "list[tuple[int, Optional[str]]]",
    killed: "set[int]",
    depth: int,
    anchor_identity: "Optional[str]" = None,
) -> "Optional[bool]":
    """Reads the children of each level just before signalling it; True if signalled, False if nothing left, None if the walk could not run (keep the record)."""
    if depth <= 0:
        # None, not False: the walk was not performed, so the record must survive.
        return None
    # Use the anchor's identity so a reused pid cannot root the snapshot at a stranger.
    found, known = _windows_collect_descendants_known(anchor, anchor_identity)
    if not known:
        return None
    progressed = False
    for child_pid, child_identity in reversed(found):
        if child_pid in killed:
            continue
        if not _signalable(child_pid) or not _pid_alive(child_pid):
            killed.add(child_pid)
            continue
        if not _provably_the_same(child_pid, child_identity):
            # Provably other is dropped; unreadable is reported so its record survives.
            if (
                _pid_alive(child_pid)
                and not _pid_is_zombie(child_pid)
                and not _provably_different(child_pid, child_identity)
            ):
                unresolved.append((child_pid, child_identity))
            killed.add(child_pid)
            continue
        # Below first: once the anchor dies its children can no longer be found.
        below = _windows_kill_below(
            child_pid, attempted, unresolved, killed, depth - 1, child_identity
        )
        if below is None:
            unresolved.append((child_pid, child_identity))
        try:
            _windows_terminate_pid(child_pid, child_identity)
        except Exception:  # noqa: BLE001 - best effort, like the rest of this
            pass
        killed.add(child_pid)
        attempted.append((child_pid, child_identity))
        progressed = True
    return progressed


# Re-walks catch children started during each taskkill's 15s ceiling.
_LATE_WALK_ROUNDS = 4


def _windows_terminate_collected(
    collected: "list[tuple[int, Optional[str]]]",
) -> "list[tuple[int, Optional[str]]]":
    """Uses taskkill /F per survivor, never /T (stale links reach recycled pids); survivors are re-read, not trusted from exit status, and keep the record."""
    attempted: "list[tuple[int, Optional[str]]]" = []
    unresolved: "list[tuple[int, Optional[str]]]" = []
    for pid, identity in reversed(collected):
        if not _signalable(pid) or not _pid_alive(pid):
            continue
        if not _provably_the_same(pid, identity):
            # Unknown identity: never signalled, always reported as unresolved.
            if (
                _pid_alive(pid)
                and not _pid_is_zombie(pid)
                and not _provably_different(pid, identity)
            ):
                unresolved.append((pid, identity))
            continue
        # Repeated, bounded rounds: late descendants can spawn during each 15s taskkill.
        killed_below: "set[int]" = set()
        still_growing = True
        for _round in range(_LATE_WALK_ROUNDS):
            progressed = _windows_kill_below(
                pid, attempted, unresolved, killed_below, _SUBTREE_CAPTURE_DEPTH, identity
            )
            if progressed is None:
                # The walk failed: still kill the survivor, but report it unresolved so the record stays.
                unresolved.append((pid, identity))
                still_growing = False
                break
            if not progressed:
                still_growing = False
                break
        if still_growing:
            # Rounds ran out with pids still appearing: report unresolved.
            unresolved.append((pid, identity))
        try:
            _windows_terminate_pid(pid, identity)
        except Exception:  # noqa: BLE001 - best effort, like the rest of this
            pass
        attempted.append((pid, identity))
    # Accounting, not signalling: alive with unreadable identity still counts as running.
    # Settle first: TerminateProcess is asynchronous.
    survivors = _survivors_after_settling(
        attempted,
        lambda pid, identity: (
            _pid_alive(pid) and not _pid_is_zombie(pid) and not _provably_different(pid, identity)
        ),
    )
    # Deepest first throughout, and deduplicated: a pid can be both unresolved and still alive, and the caller adopts each entry.
    ordered: "list[tuple[int, Optional[str]]]" = []
    seen: "set[int]" = set()
    for pid, identity in unresolved + survivors:
        if pid in seen:
            continue
        seen.add(pid)
        ordered.append((pid, identity))
    return ordered


def _windows_terminate_validated_tree(pid: int, identity: "Optional[str]" = None) -> bool:
    """Tree kill without taskkill /T (it follows stale parent links): enumerate descendants before signalling the root; False if anything survives."""
    if identity is not None and not _provably_the_same(pid, identity):
        return False
    descendants, known = _windows_collect_descendants_known(pid, identity)
    # Kill the root first, right after the snapshot, so it cannot spawn during slow taskkills.
    if _signalable(pid) and _pid_alive(pid):
        try:
            _windows_terminate_pid(pid, identity if identity is not None else _pid_identity(pid))
        except Exception:  # noqa: BLE001 - best effort, like the rest of this
            pass
    # Re-collect survivors with creation-time validation before killing: late children spawned in the 15s window are reached, recycled PIDs are not.
    survivors = _windows_terminate_collected(descendants)
    # An unenumerable tree is not empty, so this cannot answer True.
    if not known:
        return False
    # Read back the root with the same grace: TerminateProcess returns before exit.
    if _survivors_after_settling(
        [(pid, identity)],
        lambda candidate, _identity: _pid_alive(candidate) and not _pid_is_zombie(candidate),
    ):
        return False
    return not survivors


def _still_the_same(pid: int, identity: "Optional[str]") -> bool:
    """False only when the pid is provably a different process now."""
    current = _pid_identity(pid)
    if identity is None or current is None:
        return True
    return _same_identity(identity, current)


def _provably_different(pid: int, identity: "Optional[str]") -> bool:
    """True only when the pid is provably a DIFFERENT process than it was.

    The third answer `_provably_the_same` cannot give. That one folds "somebody else" and
    "cannot tell" into the same False, which is right for deciding whether to signal and
    wrong for deciding whether to report: a number that has moved on to a stranger is not
    our leaked worker and must not be adopted, while one we merely cannot read might be.
    """
    if identity is None:
        return False
    current = _pid_identity(pid)
    if current is None:
        return False
    return not _same_identity(identity, current)


def _provably_the_same(pid: int, identity: "Optional[str]") -> bool:
    """Fail-closed: an identity that cannot be read on either side means the process is left alone."""
    if identity is None:
        return False
    current = _pid_identity(pid)
    if current is None:
        return False
    return _same_identity(identity, current)


def _group_member_pids(pgid: int) -> "Optional[list[int]]":
    """Pids in a process group, or None when they cannot be enumerated."""
    if _is_linux():
        try:
            found = []
            for entry in os.listdir("/proc"):
                if not entry.isdigit():
                    continue
                try:
                    with open(f"/proc/{entry}/stat", encoding = "utf-8") as fh:
                        stat = fh.read()
                except OSError:
                    continue
                # After the comm field: state, ppid, pgrp, ...
                tail = stat[stat.rfind(")") + 2 :].split()
                if len(tail) > 2 and tail[2] == str(pgid):
                    found.append(int(entry))
            return found
        except Exception:
            return None
    if sys.platform == "darwin":
        try:
            import subprocess

            out = subprocess.run(
                ["ps", "-o", "pid=", "-g", str(pgid)],
                capture_output = True,
                text = True,
                encoding = "utf-8",
                errors = "replace",
                timeout = 5,
            )
            # ps exits nonzero on failure too; reporting empty would drop a live descendant's record.
            if out.returncode != 0:
                return None
            return [int(x) for x in (out.stdout or "").split()]
        except Exception:
            return None
    return None


# macOS has no PDEATHSIG or job objects: record children and sweep leftovers at startup.
# ── Crash-survivable child record ──
def _breadcrumb_dir():
    from pathlib import Path

    override = os.environ.get("UNSLOTH_STUDIO_CHILD_RECORD")
    if override:
        return Path(override)
    try:
        from utils.paths.storage_roots import studio_root
        return Path(studio_root()) / "run" / "children"
    except Exception:
        return None


def _breadcrumb_file():
    # One file per owner: two instances can share a home, and one shared file would let the second erase the children of the first.
    directory = _breadcrumb_dir()
    return None if directory is None else directory / f"{os.getpid()}.json"


_owner_identity: Optional[str] = None


def _own_identity() -> "Optional[str]":
    """Captured once and kept: a None would make a reused pid read as this owner still running."""
    global _owner_identity
    if _owner_identity is None:
        _owner_identity = _identity_for_record(os.getpid())
    return _owner_identity


def _refreshed_identity(pid: int, identity: "Optional[str]") -> "Optional[str]":
    """A None identity is never signalled, so fill it in while the child lives and write it back."""
    if identity is not None or not _pid_alive(pid):
        return identity
    identity = _pid_identity(pid)
    if identity is not None:
        _tracked_pids[pid] = identity
    return identity


def _write_breadcrumb() -> None:
    path = _breadcrumb_file()
    if path is None:
        return
    try:
        import json

        path.parent.mkdir(parents = True, exist_ok = True)
        payload = {
            "owner_pid": os.getpid(),
            "owner_identity": _own_identity(),
            "children": [
                {
                    "pid": pid,
                    "identity": _refreshed_identity(pid, identity),
                    "pgid": _tracked_pgids.get(pid),
                }
                for pid, identity in _tracked_pids.items()
            ],
        }
        tmp = path.with_suffix(".tmp")
        tmp.write_text(json.dumps(payload), encoding = "utf-8")
        tmp.replace(path)
    except Exception:
        pass


def clear_breadcrumb() -> None:
    """Keeps the record while any tracked child is still alive, or the next startup cannot find it."""
    with _record_lock:
        if _tracked_pids:
            _write_breadcrumb()
            return
        # Inside the lock, or a concurrent spawn's record could be deleted.
        path = _breadcrumb_file()
        if path is not None:
            _unlink(path)


def _identity_or_none(pid) -> "Optional[str]":
    return _pid_identity(pid) if isinstance(pid, int) and pid > 0 else None


def _pid_alive(pid: int) -> bool:
    if _is_windows():
        # NOT os.kill(pid, 0): that is TerminateProcess here, so the probe would kill the process it is asking about.
        try:
            import ctypes
            from ctypes import wintypes

            SYNCHRONIZE = 0x0010_0000
            PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
            WAIT_TIMEOUT = 0x102
            ERROR_ACCESS_DENIED = 5

            kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
            kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
            kernel32.OpenProcess.restype = wintypes.HANDLE
            kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD]
            kernel32.WaitForSingleObject.restype = wintypes.DWORD
            kernel32.CloseHandle.argtypes = [wintypes.HANDLE]

            handle = kernel32.OpenProcess(
                SYNCHRONIZE | PROCESS_QUERY_LIMITED_INFORMATION, False, pid
            )
            if not handle:
                # ACCESS_DENIED means alive but another user's; else it is gone.
                return _last_error(ctypes) == ERROR_ACCESS_DENIED
            try:
                # Signalled means exited; still waiting means running.
                return kernel32.WaitForSingleObject(handle, 0) == WAIT_TIMEOUT
            finally:
                kernel32.CloseHandle(handle)
        except Exception:
            return False
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return False
    except PermissionError:
        return True
    except OSError:
        return False
    return True


def pid_is_running(pid: "Optional[int]") -> bool:
    """An exited child nobody has waited on is a zombie and does not count as running."""
    if not is_signalable_pid(pid):
        return False
    return _pid_alive(pid) and not _pid_is_zombie(pid)


def _pid_is_zombie(pid: int) -> bool:
    """Exited, unreaped child: it still answers signals, so survivor checks must exclude it."""
    if _is_windows():
        return False
    if _is_linux():
        try:
            with open(f"/proc/{pid}/stat", encoding = "utf-8") as fh:
                return fh.read().rsplit(")", 1)[1].split()[0] == "Z"
        except Exception:
            return False
    try:
        import subprocess
        out = subprocess.run(
            ["ps", "-o", "state=", "-p", str(pid)],
            capture_output = True,
            text = True,
            encoding = "utf-8",
            errors = "replace",
            timeout = 5,
        )
        return (out.stdout or "").strip().startswith("Z")
    except Exception:
        return False


def _identity_for_record(pid: int, attempts: int = 3) -> Optional[str]:
    """Retries, since an identity recorded as None means the entry is never signalled."""
    import time

    for attempt in range(attempts):
        identity = _pid_identity(pid)
        if identity is not None:
            return identity
        if not _pid_alive(pid):
            break
        if attempt + 1 < attempts:
            time.sleep(0.05 * (attempt + 1))
    return None


def adopt_pid(
    pid: Optional[int],
    identity: "Optional[str]" = None,
    *,
    from_snapshot: bool = False,
) -> None:
    """Pids from an earlier snapshot must pass identity (from_snapshot=True: an unreadable one is not adopted), so a recycled number is never adopted."""
    if not _signalable(pid):
        return
    if identity is None and from_snapshot:
        # Unknown from a snapshot adopts nobody either: see `from_snapshot` above.
        return
    if identity is not None and not _provably_the_same(pid, identity):
        # Unknown adopts nobody: it may already be a stranger, and that is not recoverable.
        return
    # Without the fork handler a fork child would claim this process's children.
    _adopt_fork_reset()
    if identity is None:
        identity = _identity_for_record(pid)
    pgid = _own_process_group(pid)
    with _record_lock:
        _tracked_pids[pid] = identity
        if pgid is not None:
            _tracked_pgids[pid] = pgid
        _write_breadcrumb()
    if _is_windows() and _win_job_handle:
        try:
            import ctypes
            from ctypes import wintypes

            kernel32 = ctypes.WinDLL("kernel32", use_last_error = True)
            _win_signatures(kernel32)
            kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
            kernel32.OpenProcess.restype = wintypes.HANDLE
            PROCESS_SET_QUOTA, PROCESS_TERMINATE = 0x0100, 0x0001
            PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
            handle = kernel32.OpenProcess(
                PROCESS_SET_QUOTA | PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION,
                False,
                pid,
            )
            if handle:
                # Verify identity through the handle being assigned: the pid may have been reused, and a
                # kill-on-close job would kill a stranger. Unreadable means skip.
                opened = _windows_identity_of_handle(kernel32, handle)
                if identity is not None and opened is not None and opened == identity:
                    kernel32.AssignProcessToJobObject(_win_job_handle, handle)
                kernel32.CloseHandle(handle)
        except Exception:
            pass


# Set once when the app starts quitting, read by every spawner in the process.
_shutdown_latch = threading.Event()


def mark_process_shutting_down() -> None:
    """Process-wide latch: spawners on other objects also refuse after terminate_all takes its snapshot."""
    _shutdown_latch.set()


def is_process_shutting_down() -> bool:
    """Whether a spawn must be refused because the app is quitting."""
    return _shutdown_latch.is_set()


_quit_signal_at: Optional[float] = None


def mark_quit_signal_received() -> None:
    global _quit_signal_at
    if _quit_signal_at is None:
        _quit_signal_at = time.monotonic()


def seconds_since_quit_signal() -> Optional[float]:
    return None if _quit_signal_at is None else time.monotonic() - _quit_signal_at


def begin_process_lifecycle() -> None:
    """Clears the shutdown latch for an embedded host that calls run_server again; quitting is
    terminal for a CLI run, but in-process callers like studio/backend/colab.py reuse the
    interpreter, so an uncleared latch would refuse every spawn."""
    global _quit_signal_at
    _shutdown_latch.clear()
    _quit_signal_at = None


def terminate_all(timeout: float = 5.0) -> "list[int]":
    """Only signals pids whose identity matches; unverifiable ones are kept and returned as survivors."""
    survivors: "list[int]" = []
    # Snapshot under the write lock: adopt_pid can still run concurrently.
    with _record_lock:
        tracked = list(_tracked_pids.items())
    for pid, identity in tracked:
        with _record_lock:
            _tracked_pids.pop(pid, None)
            pgid = _tracked_pgids.pop(pid, None)
        if not _signalable(pid):
            continue
        current = _pid_identity(pid)
        if not _pid_alive(pid):
            # A leader that exited first can still have a group holding the GPU.
            if not _reap_orphaned_group(pgid, pid, timeout) and _group_has_members(pgid):
                survivors.append(pid)
                with _record_lock:
                    _tracked_pids[pid] = identity
                    if pgid is not None:
                        _tracked_pgids[pid] = pgid
            continue
        if current is not None and identity is not None and not _same_identity(identity, current):
            continue
        if identity is None or current is None:
            # Cannot prove it is ours: do not signal, but keep it recorded while alive.
            if _pid_alive(pid) and not _pid_is_zombie(pid):
                with _record_lock:
                    _tracked_pids[pid] = identity
                    # Keep the group with the pid: it is the only handle once the leader exits.
                    if pgid is not None:
                        _tracked_pgids[pid] = pgid
            continue
        tree_stands = False
        try:
            if _is_windows():
                # Kill the tree via the validated collector, never taskkill /T (stale parent links).
                tree_stands = not _windows_terminate_validated_tree(pid, identity)
            else:
                _posix_terminate(pid, timeout)
        except Exception:
            pass
        current_now = _identity_or_none(pid)
        still_ours = (
            _pid_alive(pid)
            and not _pid_is_zombie(pid)
            and current_now is not None
            and _same_identity(identity, current_now)
        )
        # Or the leader exited but its group did not: the only handle is lost, which on Windows is a tree taskkill could not take.
        if still_ours or tree_stands or _group_has_members(pgid):
            survivors.append(pid)
            with _record_lock:
                _tracked_pids[pid] = identity
                if pgid is not None:
                    _tracked_pgids[pid] = pgid
    return survivors


def terminate_pid(
    pid: "Optional[int]",
    timeout: float = 5.0,
    *,
    owner_verified: bool = False,
) -> None:
    """owner_verified waives only the cannot-prove refusal; a pid provably reused is still left alone."""
    # Floor here too: _windows_terminate_tree bypasses the POSIX helper's check.
    if not _signalable(pid):
        return
    with _record_lock:
        identity = _tracked_pids.get(pid)
        pgid = _tracked_pgids.get(pid)
    # Verify identity: an announced child can exit unannounced and its pid be reused.
    current = _pid_identity(pid)
    if identity is not None and current is not None and not _same_identity(identity, current):
        # A pid is only reusable once nothing holds the number as a process group either, so no group of ours is left to reap here.
        forget_pid(pid)
        return
    if not owner_verified and _pid_alive(pid) and (identity is None or current is None):
        # Cannot prove it is ours: leave it and keep the record for the startup sweep.
        return
    tree_stands = False
    try:
        if _is_windows():
            # Not taskkill /T: it follows stale parent links and would kill rejected strangers. False means only the leader was signalled, so keep the record.
            tree_stands = not _windows_terminate_validated_tree(pid, identity)
        else:
            _posix_terminate(pid, timeout)
            # A dead leader loses getpgid, so the recorded group is the only handle left.
            if _group_has_members(pgid):
                _reap_orphaned_group(pgid, pid, timeout)
    except Exception:  # noqa: BLE001 - best effort, like the rest of this
        pass
    if tree_stands:
        return
    forget_pid(pid)


def reap_recorded_children(timeout: float = 5.0) -> "list[int]":
    """Considers every record, so a crash while a sibling ran is still cleaned up; only identity-matched pids are signalled, never a recycled pid."""
    directory = _breadcrumb_dir()
    if directory is None or not directory.is_dir():
        return []
    killed: "list[int]" = []
    # Revisit only deferred records whose owner may die later in this sweep; never signal twice.
    pending = sorted(directory.glob("*.json"))
    for _pass in range(4):
        deferred: "list" = []
        found: "list[int]" = []
        for path in pending:
            reaped, owner_alive = _reap_one_record(path, timeout)
            found.extend(reaped)
            if owner_alive:
                deferred.append(path)
        killed.extend(found)
        if not deferred or not found:
            break
        pending = deferred
    return killed


def _reap_one_record(path, timeout: float) -> "tuple[list[int], bool]":
    """``(pids signalled, whether it was left alone because its owner is alive)``."""
    import json

    killed: "list[int]" = []
    try:
        record = json.loads(path.read_text(encoding = "utf-8"))
    except Exception:
        _unlink(path)
        return killed, False
    if not isinstance(record, dict):
        _unlink(path)
        return killed, False

    owner_pid = record.get("owner_pid")
    # Anything but the string this wrote is treated as no identity.
    owner_identity = _recorded_identity(record.get("owner_identity"))
    # Identity decides, not the pid: pids recycle.
    current_owner = _identity_or_none(owner_pid)
    owner_matches = (
        owner_identity is None
        or current_owner is None
        or _same_identity(owner_identity, current_owner)
    )
    if owner_pid == os.getpid() and owner_matches:
        return killed, False
    if (
        isinstance(owner_pid, int)
        and owner_matches
        and _pid_alive(owner_pid)
        # kill(pid, 0) succeeds for a zombie, which is still a dead owner.
        and not _pid_is_zombie(owner_pid)
    ):
        return killed, True

    unresolved = False
    children = record.get("children")
    for entry in children if isinstance(children, list) else []:
        pid = entry.get("pid") if isinstance(entry, dict) else None
        if not _signalable(pid):
            # Old records can name pid 1, and signalling that kills everything the user owns; drop rather than defer so it is gone after one startup.
            continue
        # A zombie holds nothing; signalling it would burn the grace period.
        if not _pid_alive(pid) or _pid_is_zombie(pid):
            # The group can outlive its leader (shim crashed, server holds the GPU).
            pgid = entry.get("pgid")
            if _reap_orphaned_group(pgid, pid, timeout):
                killed.append(pid)
            elif _group_has_members(pgid):
                unresolved = True
            continue
        identity = _recorded_identity(entry.get("identity"))
        current = _identity_or_none(pid)
        if identity is None or current is None:
            # Unverifiable: never signal a pid that might now be something else.
            unresolved = True
            continue
        if not _same_identity(identity, current):
            continue
        tree_stands = False
        if _is_windows():
            # Kill the tree via the validated collector, not taskkill /T, which follows stale parent links.
            tree_stands = not _windows_terminate_validated_tree(pid, identity)
        else:
            _posix_terminate(pid, timeout = timeout)
        killed.append(pid)
        if tree_stands:
            unresolved = True
        if _pid_alive(pid) and not _pid_is_zombie(pid):
            unresolved = True
        elif _group_has_members(entry.get("pgid")):
            unresolved = True

    if not unresolved:
        _unlink(path)
    if killed:
        try:
            import logging
            logging.getLogger(__name__).warning(
                "Reaped %d orphaned child process(es) left by a previous Unsloth: %s",
                len(killed),
                killed,
            )
        except Exception:
            pass
    return killed, False


def _own_process_group(pid: int) -> Optional[int]:
    """Returns the group only if the pid leads one; a shared group would take Unsloth down with it."""
    if _is_windows() or not hasattr(os, "getpgid"):
        return None
    try:
        pgid = os.getpgid(pid)
    except Exception:
        return None
    if not _signalable(pgid):
        return None
    return pgid if pgid == pid else None


def _reap_orphaned_group(pgid: object, pid: int, timeout: float) -> bool:
    """No identity check needed: the dead leader's pid stays reserved while the group still has members."""
    if not _signalable(pgid) or pgid != pid or _is_windows() or not hasattr(os, "killpg"):
        return False
    try:
        os.killpg(pgid, 0)
    except Exception:
        return False
    # Zombies answer the probe where pid 1 does not reap; skip empty groups.
    if not _group_has_members(pgid):
        return False
    import time

    try:
        os.killpg(pgid, signal.SIGTERM)
    except Exception:
        return False
    deadline = time.monotonic() + max(0.0, timeout)
    next_state_check = 0.0
    while time.monotonic() < deadline:
        try:
            os.killpg(pgid, 0)
        except Exception:
            return True
        now = time.monotonic()
        if now >= next_state_check:
            # Where pid 1 does not reap, exited members stay zombies; recheck state periodically.
            next_state_check = now + 0.5
            if not _group_has_members(pgid):
                return True
        time.sleep(0.1)
    try:
        os.killpg(pgid, signal.SIGKILL)
    except Exception:
        pass
    # Only gone counts: the record is the last handle on whatever holds the GPU.
    try:
        os.killpg(pgid, 0)
        return False
    except Exception:
        return True


def _windows_terminate_pid(pid: int, identity: "Optional[str]" = None) -> bool:
    """Kills one pid with taskkill /F, no /T; caller verifies identity, and the fallback re-checks it and is skipped without one, since the pid can be reused."""
    import subprocess

    # Terminate through a handle first: taskkill /PID re-resolves the number and can hit a reused pid.
    through_handle = _windows_terminate_through_a_handle(pid, identity)
    if through_handle is not None:
        return through_handle

    # No handle: re-verify identity right before taskkill, or skip; a leak is recoverable, a wrong kill is not.
    if not _provably_the_same(pid, identity):
        return False

    try:
        completed = subprocess.run(
            ["taskkill", "/PID", str(pid), "/F"],
            capture_output = True,
            timeout = 15,
            creationflags = getattr(subprocess, "CREATE_NO_WINDOW", 0),
        )
        # 128 is "already gone", as in the tree call below.
        if completed.returncode in (0, 128):
            return True
    except (OSError, subprocess.SubprocessError):
        pass
    # Re-check: taskkill can take 15s and the pid may be freed meanwhile.
    if not _provably_the_same(pid, identity):
        return False
    try:
        os.kill(pid, signal.SIGTERM)
    except OSError:
        pass
    return False


def _windows_terminate_tree(pid: int) -> bool:
    """Runs taskkill /T /F; caller must have verified identity; False means workers may still run, so the caller keeps the record."""
    import subprocess

    try:
        completed = subprocess.run(
            ["taskkill", "/PID", str(pid), "/T", "/F"],
            capture_output = True,
            timeout = 15,
            creationflags = getattr(subprocess, "CREATE_NO_WINDOW", 0),
        )
        # check=False does not raise; 128 is already gone.
        if completed.returncode in (0, 128):
            return True
    except (OSError, subprocess.SubprocessError):
        pass
    try:
        os.kill(pid, signal.SIGTERM)
    except OSError:
        pass
    return False


def _unlink(path) -> None:
    try:
        path.unlink(missing_ok = True)
    except Exception:
        pass


def _posix_terminate(pid: int, timeout: float = 5.0) -> None:
    # SIGTERM, wait timeout, then SIGKILL; prefer the group when pid leads one.
    if not _signalable(pid):
        return
    group_leader = False
    try:
        group_leader = os.getpgid(pid) == pid
    except Exception:
        pass
    # Same-group children are unreachable by killpg; collect them now while the parent lives.
    descendants = [] if group_leader else collect_descendants(pid)
    try:
        _posix_terminate_one(pid, group_leader, timeout)
    finally:
        terminate_descendants(descendants, timeout)


def _posix_terminate_one(pid: int, group_leader: bool, timeout: float) -> None:
    # Belt and braces: killpg(1) would reach every process the user owns.
    if not _signalable(pid):
        return
    killer = os.killpg if group_leader else os.kill
    try:
        killer(pid, signal.SIGTERM)
    except ProcessLookupError:
        return
    except Exception:
        return
    deadline = time.monotonic() + max(0.0, timeout)
    next_state_check = 0.0
    while time.monotonic() < deadline:
        try:
            killer(pid, 0)
        except ProcessLookupError:
            return
        except Exception:
            break
        now = time.monotonic()
        if now >= next_state_check:
            # A zombie answers signal 0 like a live process; check state periodically (forks off Linux).
            next_state_check = now + 0.5
            gone = (not _group_has_members(pid)) if group_leader else _pid_is_zombie(pid)
            if gone:
                return
        time.sleep(0.05)
    try:
        killer(pid, signal.SIGKILL)
    except Exception:
        pass
    _settle_after_the_kill(pid, group_leader)