Skip to content

💫 Toolkit to help you get started with SDD or any other process!

18.7 KB
Raw Blame
"""Bounded capture of an integration CLI's stdout and stderr.

``subprocess.run(capture_output=True)`` buffers everything a process writes,
decodes it with the locale encoding, and loses the output on a timeout. Captured
workflow dispatch needs the opposite on all three: a per-stream byte cap, a
fixed UTF-8 decoding, and whatever was read before a timeout or interruption.
"""

from __future__ import annotations

import codecs
import os
import signal
import subprocess
import threading
import time
from collections.abc import Iterator, Sequence
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Any

DEFAULT_CAPTURE_MAX_BYTES = 1024 * 1024
UNLIMITED_CAPTURE = -1

_READ_CHUNK_BYTES = 64 * 1024
# After a kill, a descendant that inherited the pipes can keep them open; wait
# this long for end-of-file before returning what has been read so far.
_DRAIN_GRACE_SECONDS = 2.0


def capture_max_bytes_is_valid(value: Any) -> bool:
    """Return whether *value* is a positive int or ``-1`` (unlimited).

    bool is a subclass of int, but ``capture_max_bytes: true`` is a config
    error rather than a one-byte limit, so it is rejected explicitly.
    """
    if isinstance(value, bool) or not isinstance(value, int):
        return False
    return value > 0 or value == UNLIMITED_CAPTURE


@dataclass(frozen=True)
class CapturedProcess:
    """Outcome of one captured process.

    ``stdout_truncated``/``stderr_truncated`` report size-based truncation
    only. Output can also be incomplete because the process was stopped;
    ``timed_out`` and ``interrupted`` report that separately.
    """

    exit_code: int
    stdout: str
    stderr: str
    stdout_truncated: bool = False
    stderr_truncated: bool = False
    timed_out: bool = False
    interrupted: bool = False

    def as_dispatch_result(self) -> dict[str, Any]:
        """Return the ``dispatch_command`` result dict for this process."""
        return {
            "exit_code": self.exit_code,
            "stdout": self.stdout,
            "stderr": self.stderr,
            "stdout_truncated": self.stdout_truncated,
            "stderr_truncated": self.stderr_truncated,
            "timed_out": self.timed_out,
            "interrupted": self.interrupted,
        }


class _Sink:
    """Retain at most *limit* bytes of a stream; count the rest as truncated."""

    def __init__(self, limit: int) -> None:
        self._limit = limit
        self._data = bytearray()
        self._truncated = False
        self._lock = threading.Lock()

    def feed(self, chunk: bytes) -> None:
        with self._lock:
            if self._limit != UNLIMITED_CAPTURE:
                room = self._limit - len(self._data)
                if len(chunk) > room:
                    self._truncated = True
                    chunk = chunk[:room]
            self._data += chunk

    def snapshot(self) -> tuple[bytes, bool]:
        with self._lock:
            return bytes(self._data), self._truncated


def _drain(pipe: Any, sink: _Sink) -> None:
    """Read *pipe* to end-of-file, including past the sink's limit.

    Reading continues after the cap so the process never blocks writing to a
    full pipe; the sink discards what it does not retain.
    """
    try:
        while chunk := pipe.read1(_READ_CHUNK_BYTES):
            sink.feed(chunk)
    except (OSError, ValueError):
        pass
    finally:
        try:
            pipe.close()
        except OSError:
            pass


# How often a reader join checks whether the capture was cancelled.
_JOIN_POLL_SECONDS = 0.1


def _join(
    readers: Sequence[threading.Thread],
    deadline: float | None,
    cancelled: Any = None,
) -> bool:
    """Join *readers* until *deadline*; return whether all reached end-of-file.

    When *cancelled* is given, it is polled while waiting, and the join stops
    early, returning False, once it returns true.
    """
    for reader in readers:
        while reader.is_alive():
            if cancelled is not None and cancelled():
                return False
            if deadline is None:
                wait = None if cancelled is None else _JOIN_POLL_SECONDS
            else:
                remaining = deadline - time.monotonic()
                if remaining <= 0:
                    return False
                wait = (
                    remaining
                    if cancelled is None
                    else min(remaining, _JOIN_POLL_SECONDS)
                )
            reader.join(wait)
    return True


# Windows process creation flag; not exported by ``subprocess``.
_CREATE_SUSPENDED = 0x00000004


class _ProcessTree:
    """Run a process so that it and its process group or job can be killed.

    Killing only the CLI would leave the tools it started running, still
    changing the project and holding the pipes open after the step has
    failed. On POSIX the CLI leads a new session and its process group is
    killed; on Windows it runs inside a job object that is terminated. The
    boundary is that group or job, not every descendant: a process that
    moves to another session or process group (POSIX) or breaks away from
    the job (Windows), as a daemonising helper can, survives the kill.
    A separate session also keeps the terminal's Ctrl+C from reaching the CLI
    directly: the interrupt is handled here, by killing the group or job.
    """

    def __init__(self, args: Sequence[str], cwd: str | None) -> None:
        isolation: dict[str, Any] = (
            {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP | _CREATE_SUSPENDED}
            if os.name == "nt"
            else {"start_new_session": True}
        )
        self.proc = subprocess.Popen(
            args,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=cwd,
            **isolation,
        )
        self._job: Any = None
        if os.name == "nt":
            try:
                self._job = _start_in_job(self.proc)
            except BaseException:
                # Fail closed: a CLI whose descendants could not be killed
                # never runs.
                self.proc.kill()
                self.proc.wait()
                for pipe in (self.proc.stdout, self.proc.stderr):
                    if pipe is not None:
                        pipe.close()
                raise

        self._lock = threading.Lock()
        # Set when another thread kills the tree through interrupt().
        self.interrupted = False

    def _send_kill(self) -> None:
        if self._job is not None:
            _windows().TerminateJobObject(self._job, 1)
        elif os.name != "nt":
            try:
                os.killpg(self.proc.pid, signal.SIGKILL)
            except OSError:
                pass
        try:
            self.proc.kill()
        except OSError:
            pass

    def interrupt(self) -> None:
        """Kill the process group or job from another thread.

        The thread running the capture sees the process exit and reports
        the capture as interrupted.
        """
        with self._lock:
            self.interrupted = True
            self._send_kill()

    def kill(self) -> None:
        """Kill the process group or job, then reap the process."""
        with self._lock:
            self._send_kill()
        self.proc.wait()

    def close(self) -> None:
        with self._lock:
            if self._job is not None:
                _windows().CloseHandle(self._job)
                self._job = None


# Captures running in any thread, so an interrupt of the main thread can
# reach the ones running in worker threads; see cancel_captures().
_active_lock = threading.Lock()
_active: set[_ProcessTree] = set()
_cancelling = 0


def _register(tree: _ProcessTree) -> None:
    with _active_lock:
        _active.add(tree)
        cancelling = _cancelling > 0
    if cancelling:
        tree.interrupt()


def _unregister(tree: _ProcessTree) -> None:
    with _active_lock:
        _active.discard(tree)


@contextmanager
def cancel_captures() -> Iterator[None]:
    """Kill every running capture, and any that starts, until the block ends.

    Ctrl+C raises ``KeyboardInterrupt`` in the main thread only, and a
    captured CLI runs in its own session or job, so it does not receive the
    terminal's Ctrl+C either. When the main thread is interrupted while
    worker threads run captures, wait for those workers inside this block:
    their captures are killed and report ``interrupted``, and a capture that
    starts meanwhile is killed as soon as it is registered.
    """
    global _cancelling
    with _active_lock:
        _cancelling += 1
        trees = list(_active)
    try:
        for tree in trees:
            tree.interrupt()
        yield
    finally:
        with _active_lock:
            _cancelling -= 1


@contextmanager
def _deferred_interrupt() -> Iterator[list[int]]:
    """Record Ctrl+C instead of raising it until cleanup is in place.

    A ``KeyboardInterrupt`` raised after ``Popen`` returned but before the
    code that kills the process on interruption would leave the CLI running
    in its own session (POSIX) or suspended outside its job (Windows).
    Within this block a SIGINT is appended to the yielded list; the caller
    treats a non-empty list as an interruption. Only the main thread
    receives SIGINT, and only Python's default handler is replaced, so a
    custom or ignoring handler keeps its behaviour.
    """
    pending: list[int] = []
    if (
        threading.current_thread() is not threading.main_thread()
        or signal.getsignal(signal.SIGINT) is not signal.default_int_handler
    ):
        yield pending
        return
    previous = signal.signal(
        signal.SIGINT, lambda signum, frame: pending.append(signum)
    )
    try:
        yield pending
    finally:
        signal.signal(signal.SIGINT, previous)


_kernel32: Any = None


def _windows() -> Any:
    """Return kernel32 with the signatures the job-object helpers use."""
    global _kernel32
    if _kernel32 is None:
        import ctypes
        from ctypes import wintypes

        kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
        signatures = {
            "CreateJobObjectW": ((wintypes.LPVOID, wintypes.LPCWSTR), wintypes.HANDLE),
            "AssignProcessToJobObject": (
                (wintypes.HANDLE, wintypes.HANDLE),
                wintypes.BOOL,
            ),
            "TerminateJobObject": ((wintypes.HANDLE, wintypes.UINT), wintypes.BOOL),
            "CreateToolhelp32Snapshot": (
                (wintypes.DWORD, wintypes.DWORD),
                wintypes.HANDLE,
            ),
            "Thread32First": ((wintypes.HANDLE, ctypes.c_void_p), wintypes.BOOL),
            "Thread32Next": ((wintypes.HANDLE, ctypes.c_void_p), wintypes.BOOL),
            "OpenThread": (
                (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD),
                wintypes.HANDLE,
            ),
            "ResumeThread": ((wintypes.HANDLE,), wintypes.DWORD),
            "CloseHandle": ((wintypes.HANDLE,), wintypes.BOOL),
        }
        for name, (argtypes, restype) in signatures.items():
            function = getattr(kernel32, name)
            function.argtypes = argtypes
            function.restype = restype
        _kernel32 = kernel32
    return _kernel32


def _start_in_job(proc: subprocess.Popen) -> Any:
    """Put the suspended *proc* in a new job object, then resume it.

    *proc* was created suspended, so it cannot start a descendant before it
    is in the job; every process it starts afterwards joins the job too.
    Returns the job handle. Raises OSError if any step fails.
    """
    import ctypes

    kernel32 = _windows()
    job = kernel32.CreateJobObjectW(None, None)
    if not job:
        raise ctypes.WinError(ctypes.get_last_error())
    try:
        if not kernel32.AssignProcessToJobObject(job, int(proc._handle)):
            raise ctypes.WinError(ctypes.get_last_error())
        _resume_threads(proc.pid)
    except BaseException:
        # The CLI may already be running if the failure or interruption came
        # after a thread was resumed: kill everything in the job before the
        # handle goes, because closing it does not.
        kernel32.TerminateJobObject(job, 1)
        kernel32.CloseHandle(job)
        raise
    return job


def _resume_threads(pid: int) -> None:
    """Resume the threads of the suspended process *pid*.

    ``subprocess`` closes the main thread's handle, so the thread is found
    through a thread snapshot.
    """
    import ctypes
    from ctypes import wintypes

    class THREADENTRY32(ctypes.Structure):
        _fields_ = [
            ("dwSize", wintypes.DWORD),
            ("cntUsage", wintypes.DWORD),
            ("th32ThreadID", wintypes.DWORD),
            ("th32OwnerProcessID", wintypes.DWORD),
            ("tpBasePri", wintypes.LONG),
            ("tpDeltaPri", wintypes.LONG),
            ("dwFlags", wintypes.DWORD),
        ]

    th32cs_snapthread = 0x00000004
    thread_suspend_resume = 0x0002
    resume_failed = 0xFFFFFFFF
    invalid_handle = wintypes.HANDLE(-1).value

    kernel32 = _windows()
    snapshot = kernel32.CreateToolhelp32Snapshot(th32cs_snapthread, 0)
    if not snapshot or snapshot == invalid_handle:
        raise ctypes.WinError(ctypes.get_last_error())
    resumed = 0
    try:
        entry = THREADENTRY32()
        entry.dwSize = ctypes.sizeof(THREADENTRY32)
        more = kernel32.Thread32First(snapshot, ctypes.byref(entry))
        while more:
            if entry.th32OwnerProcessID == pid:
                thread = kernel32.OpenThread(
                    thread_suspend_resume, False, entry.th32ThreadID
                )
                if not thread:
                    raise ctypes.WinError(ctypes.get_last_error())
                try:
                    if kernel32.ResumeThread(thread) == resume_failed:
                        raise ctypes.WinError(ctypes.get_last_error())
                finally:
                    kernel32.CloseHandle(thread)
                resumed += 1
            more = kernel32.Thread32Next(snapshot, ctypes.byref(entry))
    finally:
        kernel32.CloseHandle(snapshot)
    if not resumed:
        raise OSError(f"Could not find the threads of process {pid} to resume.")


def _decode(data: bytes, *, complete: bool) -> str:
    """Decode captured bytes as UTF-8, replacing invalid sequences with U+FFFD.

    When the capture is not *complete* (cut at the byte limit, or the process
    was stopped), a trailing partial multibyte sequence is dropped rather than
    replaced, so a cut never invents a replacement character. Line endings are
    normalised to ``\\n`` as ``subprocess`` text mode does.
    """
    decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
    text = decoder.decode(data, final=complete)
    return text.replace("\r\n", "\n").replace("\r", "\n")


def run_captured(
    args: Sequence[str],
    *,
    cwd: str | None = None,
    timeout: float | None = None,
    max_bytes: int = DEFAULT_CAPTURE_MAX_BYTES,
) -> CapturedProcess:
    """Run *args* and capture stdout and stderr, each capped at *max_bytes*.

    *max_bytes* counts raw bytes before decoding; ``-1`` disables the cap. A
    timeout or ``KeyboardInterrupt`` kills the process and is reported on the
    result (exit code ``-1`` or ``130``) together with the output read so far,
    instead of being raised, after the process's group (POSIX) or job
    (Windows) is killed; a descendant that left it is not killed. ``OSError``
    from starting the process propagates.
    """
    if not capture_max_bytes_is_valid(max_bytes):
        raise ValueError(
            f"max_bytes must be a positive integer or -1, got {max_bytes!r}."
        )

    tree: _ProcessTree | None = None
    sinks = (_Sink(max_bytes), _Sink(max_bytes))
    readers: list[threading.Thread] = []
    deadline = None if timeout is None else time.monotonic() + timeout
    timed_out = interrupted = False
    try:
        try:
            with _deferred_interrupt() as pending_interrupt:
                tree = _ProcessTree(args, cwd)
                _register(tree)
                for pipe, sink in zip((tree.proc.stdout, tree.proc.stderr), sinks):
                    reader = threading.Thread(
                        target=_drain, args=(pipe, sink), daemon=True
                    )
                    reader.start()
                    readers.append(reader)
            # From here a Ctrl+C raises KeyboardInterrupt inside this try.
            if pending_interrupt:
                raise KeyboardInterrupt
            try:
                tree.proc.wait(timeout=timeout)
            except subprocess.TimeoutExpired:
                timed_out = True
            else:
                # The process can exit while a descendant still holds the
                # pipes open; the same deadline bounds the wait for
                # end-of-file.
                # cancel_captures() kills the CLI, but a detached helper can
                # still hold the pipes, so this wait also stops on it.
                timed_out = not _join(readers, deadline, lambda: tree.interrupted)
        except KeyboardInterrupt:
            interrupted = True
            if tree is None:
                # Setup was interrupted, and the constructor has already
                # killed the process it started.
                return CapturedProcess(
                    exit_code=130, stdout="", stderr="", interrupted=True
                )
        except BaseException:
            if tree is not None:
                tree.kill()
            raise
        if tree.interrupted:
            # Killed through cancel_captures() while this thread waited.
            interrupted, timed_out = True, False

        if timed_out or interrupted:
            # Killing the group or job closes the write ends of the pipes its
            # processes hold, so the readers reach end-of-file. Only a
            # descendant that left the group or job could keep them open; the
            # grace period bounds that wait.
            tree.kill()
            _join(readers, time.monotonic() + _DRAIN_GRACE_SECONDS)
            # A pipe whose reader never started is closed here instead.
            for pipe in (tree.proc.stdout, tree.proc.stderr)[len(readers) :]:
                pipe.close()
    finally:
        if tree is not None:
            _unregister(tree)
            tree.close()

    (stdout, stdout_truncated), (stderr, stderr_truncated) = (
        sink.snapshot() for sink in sinks
    )
    stopped = timed_out or interrupted
    if interrupted:
        exit_code = 130
    elif timed_out:
        exit_code = -1
    else:
        exit_code = tree.proc.returncode
    return CapturedProcess(
        exit_code=exit_code,
        stdout=_decode(stdout, complete=not (stopped or stdout_truncated)),
        stderr=_decode(stderr, complete=not (stopped or stderr_truncated)),
        stdout_truncated=stdout_truncated,
        stderr_truncated=stderr_truncated,
        timed_out=timed_out,
        interrupted=interrupted,
    )