Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

30.2 KB
Raw Blame
"""Typed runtime adapters for managed plugin worker packages."""

from __future__ import annotations

import asyncio
from contextvars import ContextVar
from dataclasses import asdict, dataclass
import json
import logging
import os
from pathlib import Path
import re
import subprocess
import threading
import time
from typing import Any, Mapping

from deeptutor.core.capability_protocol import (
    CapabilityManifest,
    StreamBusProtocol,
    TurnCapability,
)
from deeptutor.core.context import UnifiedContext
from deeptutor.core.stream import StreamEvent, StreamEventType
from deeptutor.core.tool_protocol import (
    BaseTool,
    ToolDefinition,
    ToolParameter,
    ToolResult,
)
from deeptutor.plugins.manifest import PluginManifestData
from deeptutor.plugins.registry import (
    PluginInstallation,
    PluginRecord,
    PluginRegistry,
    PluginStateError,
    _manifest_files,
    get_plugin_registry,
)

logger = logging.getLogger(__name__)
_worker_cancellation: ContextVar[threading.Event | None] = ContextVar(
    "worker_cancellation", default=None
)


class PluginRuntimeError(RuntimeError):
    """Raised when an enabled worker violates the plugin runtime contract."""


_WORKER_MODULE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)+$")
_PERMISSION_KEYS = {"reading", "learning_events", "network", "models", "storage", "ui"}
_HTTP_STATUS_VALUES = frozenset({200, 201, 202, 204, 400, 404, 409, 422, 500})
_HTTP_HEADER_VALUES = re.compile(r"^(?:no-store|no-cache|max-age=[0-9]{1,8})$")


@dataclass(frozen=True, slots=True)
class PluginHttpResponse:
    status: int
    body: Any = None
    headers: Mapping[str, str] | None = None


class _WorkerClient:
    def __init__(
        self,
        *,
        python_path: Path,
        venv_path: Path,
        module: str,
        permissions: dict[str, tuple[str, ...]],
        guard=None,
    ) -> None:
        if _WORKER_MODULE.fullmatch(module) is None:
            raise PluginRuntimeError(f"invalid worker module {module!r}")
        self.python_path = python_path
        self.venv_path = venv_path
        self.module = module
        self.permissions = permissions
        self.guard = guard

    def request(self, operation: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
        if self.guard is not None:
            self.guard()
        request = {"operation": operation, **(payload or {})}
        try:
            completed = _run_worker(
                [str(self.python_path), "-m", self.module],
                input=json.dumps(request, ensure_ascii=False, default=str),
                env=_clean_env(),
                cwd=self.venv_path,
            )
        except (OSError, subprocess.TimeoutExpired) as exc:
            raise PluginRuntimeError(f"plugin worker failed: {exc}") from exc
        if completed.returncode != 0:
            detail = (completed.stderr or completed.stdout or "").strip()[-2000:]
            raise PluginRuntimeError(f"plugin worker exited {completed.returncode}: {detail}")
        try:
            response = json.loads(completed.stdout)
        except json.JSONDecodeError as exc:
            raise PluginRuntimeError("plugin worker returned invalid JSON") from exc
        if not isinstance(response, dict):
            raise PluginRuntimeError("plugin worker response must be an object")
        if response.get("error") is not None:
            detail = str(response["error"])[-2000:]
            raise PluginRuntimeError(f"plugin worker error: {detail}")
        self.validate_permissions(response.get("permissions"))
        return response

    async def request_async(self, operation: str, payload: dict[str, Any]):
        """Legacy buffered workers also stop when their owning turn is cancelled."""
        cancelled = threading.Event()
        token = _worker_cancellation.set(cancelled)
        task = asyncio.create_task(asyncio.to_thread(self.request, operation, payload))
        _worker_cancellation.reset(token)
        try:
            return await asyncio.shield(task)
        except asyncio.CancelledError:
            cancelled.set()
            await asyncio.gather(task, return_exceptions=True)
            raise

    async def stream(self, operation: str, payload: dict[str, Any]):
        """NDJSON v1: bounded live events, explicit completion and process cleanup."""

        if self.guard is not None:
            self.guard()
        encoded = json.dumps(
            {"operation": operation, "transport": "ndjson-v1", **payload},
            ensure_ascii=False,
            default=str,
        ).encode()
        if len(encoded) > 1024 * 1024:
            raise PluginRuntimeError("Plugin worker request exceeds 1 MiB.")
        process = await asyncio.create_subprocess_exec(
            str(self.python_path),
            "-m",
            self.module,
            stdin=asyncio.subprocess.PIPE,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
            env=_clean_env(),
            cwd=self.venv_path,
            start_new_session=os.name != "nt",
            limit=1024 * 1024,
        )
        stderr = bytearray()

        async def drain_errors():
            while chunk := await process.stderr.read(8192):
                stderr.extend(chunk)
                if len(stderr) > 65536:
                    raise PluginRuntimeError("Plugin worker stderr exceeds 64 KiB.")

        async def produce():
            process.stdin.write(encoded)
            await process.stdin.drain()
            process.stdin.close()
            total = 0
            completed = False
            while line := await process.stdout.readline():
                total += len(line)
                if total > 16 * 1024 * 1024:
                    raise PluginRuntimeError("Plugin worker stream exceeds 16 MiB.")
                if self.guard is not None:
                    self.guard()
                try:
                    row = json.loads(line)
                except (ValueError, UnicodeError) as exc:
                    raise PluginRuntimeError("Plugin worker returned invalid NDJSON.") from exc
                if not isinstance(row, dict) or completed:
                    raise PluginRuntimeError("Invalid plugin stream envelope/order.")
                self.validate_permissions(row.get("permissions"))
                if row.get("error") is not None:
                    raise PluginRuntimeError(str(row["error"])[-2000:])
                if row.get("kind") == "complete":
                    completed = True
                elif row.get("kind") == "event" and isinstance(row.get("event"), dict):
                    await queue.put(row["event"])
                else:
                    raise PluginRuntimeError("Unknown plugin stream envelope.")
            code = await process.wait()
            if code or not completed:
                raise PluginRuntimeError(
                    f"Plugin stream ended without successful completion ({code}): "
                    + stderr.decode("utf-8", "replace")[-2000:]
                )

        queue: asyncio.Queue = asyncio.Queue(maxsize=64)
        consumer_closed = False

        async def run():
            try:
                async with asyncio.timeout(300):
                    async with asyncio.TaskGroup() as group:
                        group.create_task(drain_errors())
                        group.create_task(produce())
            finally:
                # Wake the consumer even when the worker fails with an empty stream.
                # Python 3.11/3.12 TaskGroup failures leave an internal cancellation
                # count behind. Only consumer shutdown may suppress this signal.
                if not consumer_closed:
                    await queue.put(None)

        task = asyncio.create_task(run())
        try:
            while (item := await queue.get()) is not None:
                yield item
            await task
        finally:
            consumer_closed = True
            task.cancel()
            # Kill the entire session, including render helpers inheriting pipes.
            from deeptutor.runtime.process import kill_process_tree

            await asyncio.to_thread(kill_process_tree, process.pid)
            await process.wait()
            await asyncio.gather(task, return_exceptions=True)

    def validate_permissions(self, raw: Any) -> None:
        if raw is None:
            return
        if not isinstance(raw, dict):
            raise PluginRuntimeError("worker permissions must be an object")
        for scope, values in raw.items():
            if scope not in _PERMISSION_KEYS or not isinstance(values, list):
                raise PluginRuntimeError(f"worker declares invalid permission scope {scope!r}")
            if not all(isinstance(value, str) for value in values):
                raise PluginRuntimeError(f"worker declares invalid {scope} permissions")
            allowed = set(self.permissions.get(scope, ()))
            if not set(values).issubset(allowed):
                raise PluginRuntimeError(f"worker declares unapproved {scope} permissions")


class PluginWorkerTool(BaseTool):
    deferred = True

    def __init__(
        self,
        *,
        manifest: PluginManifestData,
        installation: PluginInstallation,
        extension_id: str,
        registry: PluginRegistry | None = None,
    ) -> None:
        extension = _extension(manifest, "tool", extension_id)
        self._client = _WorkerClient(
            python_path=installation.python_path,
            venv_path=installation.venv_path,
            module=extension.entry_point,
            permissions=dict(manifest.permissions.scopes),
            guard=_managed_guard(manifest, installation, registry),
        )
        described = self._client.request("describe_tool")
        definition = described.get("definition")
        if not isinstance(definition, dict) or definition.get("name") != extension_id:
            raise PluginRuntimeError(f"worker tool declaration does not match {extension_id!r}")
        self._definition = _tool_definition(definition)

    def get_definition(self) -> ToolDefinition:
        return self._definition

    async def execute(self, **kwargs: Any) -> ToolResult:
        response = await self._client.request_async("execute_tool", {"arguments": kwargs})
        result = response.get("result")
        if not isinstance(result, dict):
            raise PluginRuntimeError("execute_tool response requires a result object")
        return ToolResult(
            content=str(result.get("content", "")),
            sources=list(result.get("sources", [])),
            metadata=dict(result.get("metadata", {})),
            success=bool(result.get("success", True)),
        )


class PluginWorkerHttpRoute:
    def __init__(
        self,
        *,
        manifest: PluginManifestData,
        installation: PluginInstallation,
        extension_id: str,
        registry: PluginRegistry | None = None,
    ) -> None:
        extension = _extension(manifest, "http_route", extension_id)
        self.extension = extension
        self._client = _WorkerClient(
            python_path=installation.python_path,
            venv_path=installation.venv_path,
            module=extension.entry_point,
            permissions=dict(manifest.permissions.scopes),
            guard=_managed_guard(manifest, installation, registry),
        )

    def handle(
        self,
        *,
        method: str,
        path: str,
        query: Mapping[str, tuple[str, ...] | list[str]],
        body: Any = None,
    ) -> PluginHttpResponse:
        if method not in self.extension.methods:
            raise PluginRuntimeError(f"HTTP method {method!r} is not declared")
        if path != self.extension.path:
            raise PluginRuntimeError("HTTP route path does not match its manifest")
        response = self._client.request(
            "handle_http",
            {
                "http": {
                    "method": method,
                    "path": path,
                    "query": {name: list(values) for name, values in query.items()},
                    "body": body,
                    "auth": self.extension.auth,
                }
            },
        )
        return _http_response(response.get("http"))


class PluginWorkerCapability(TurnCapability):
    def __init__(
        self,
        *,
        manifest: PluginManifestData,
        installation: PluginInstallation,
        extension_id: str,
        registry: PluginRegistry | None = None,
    ) -> None:
        self._clone_args = dict(
            manifest=manifest, installation=installation, extension_id=extension_id
        )
        extension = _extension(manifest, "capability", extension_id)
        self._client = _WorkerClient(
            python_path=installation.python_path,
            venv_path=installation.venv_path,
            module=extension.entry_point,
            permissions=dict(manifest.permissions.scopes),
            guard=_managed_guard(manifest, installation, registry),
        )
        described = self._client.request("describe_capability")
        self._streaming = described.get("transport") == "ndjson-v1"
        raw = described.get("capability")
        if not isinstance(raw, dict) or raw.get("name") != extension_id:
            raise PluginRuntimeError(
                f"worker capability declaration does not match {extension_id!r}"
            )
        self.manifest = CapabilityManifest(
            name=str(raw.get("name")),
            description=str(raw.get("description", "")),
            stages=[str(item) for item in raw.get("stages", [])],
            tools_used=[str(item) for item in raw.get("tools_used", [])],
            cli_aliases=[str(item) for item in raw.get("cli_aliases", [])],
            request_schema=dict(raw.get("request_schema", {})),
            config_defaults=dict(raw.get("config_defaults", {})),
        )

    def clone(self):
        return PluginWorkerCapability(**self._clone_args)

    async def run(self, context: UnifiedContext, stream: StreamBusProtocol) -> None:
        payload = {
            "context": {
                "session_id": context.session_id,
                "user_message": context.user_message,
                "conversation_history": context.conversation_history,
                "enabled_tools": context.enabled_tools,
                "active_capability": context.active_capability,
                "knowledge_bases": context.knowledge_bases,
                "attachments": [asdict(item) for item in context.attachments],
                "config_overrides": context.config_overrides,
                "language": context.language,
                "metadata": context.metadata,
            }
        }
        if self._streaming:
            async for raw in self._client.stream("run_capability", payload):
                await stream.emit(_stream_event(raw, source=self.manifest.name))
            return
        response = await self._client.request_async("run_capability", payload)
        events = response.get("events", [])
        if not isinstance(events, list):
            raise PluginRuntimeError("run_capability events must be an array")
        for raw in events:
            await stream.emit(_stream_event(raw, source=self.manifest.name))


def load_enabled_workers(
    registry: PluginRegistry | None = None,
) -> tuple[BaseTool | TurnCapability, ...]:
    """Build tool/capability adapters only for approved managed installations."""
    registry = registry or get_plugin_registry()
    records = runtime_plugin_records(registry)
    extensions: list[BaseTool | TurnCapability] = []
    for record in records:
        if record.status != "enabled" or record.manifest is None or record.installation is None:
            continue
        for extension in record.manifest.extensions:
            try:
                if extension.type == "tool":
                    extensions.append(
                        PluginWorkerTool(
                            manifest=record.manifest,
                            installation=record.installation,
                            extension_id=extension.id,
                            registry=registry,
                        )
                    )
                elif extension.type == "capability":
                    extensions.append(
                        PluginWorkerCapability(
                            manifest=record.manifest,
                            installation=record.installation,
                            extension_id=extension.id,
                            registry=registry,
                        )
                    )
            except (PluginRuntimeError, ValueError, TypeError, KeyError) as exc:
                logger.warning(
                    "Managed %s %r failed its worker contract; skipping",
                    extension.type,
                    extension.id,
                    exc_info=exc,
                )
    return tuple(extensions)


def load_enabled_tools(registry: PluginRegistry | None = None) -> tuple[BaseTool, ...]:
    return tuple(item for item in load_enabled_workers(registry) if isinstance(item, BaseTool))


def load_enabled_capabilities(
    registry: PluginRegistry | None = None,
) -> tuple[TurnCapability, ...]:
    return tuple(
        item for item in load_enabled_workers(registry) if isinstance(item, TurnCapability)
    )


def entry_point_allowed(
    registry: PluginRegistry,
    *,
    extension_type: str,
    entry_point: str,
) -> bool:
    """Gate only packages that declare this extension in a root manifest.

    Existing built-in and pre-v1 extension points keep working during the
    migration window. Once a package ships a root manifest, its exact
    extension, compatibility, approval digest, and local enablement state must
    all match before the entry point can load.
    """
    for record in registry.list_plugins(include_catalog=False):
        if record.manifest is None:
            continue
        declared = any(
            extension.type == extension_type and extension.entry_point == entry_point
            for extension in record.manifest.extensions
        )
        if declared:
            return record.status == "enabled"
    return True


def _extension(manifest: PluginManifestData, extension_type: str, extension_id: str):
    for extension in manifest.extensions:
        if extension.type == extension_type and extension.id == extension_id:
            return extension
    raise PluginRuntimeError(f"manifest does not declare {extension_type} {extension_id!r}")


def _tool_definition(raw: dict[str, Any]) -> ToolDefinition:
    parameters = []
    for item in raw.get("parameters", []):
        if not isinstance(item, dict) or not isinstance(item.get("name"), str):
            raise PluginRuntimeError("tool parameter requires a name")
        parameters.append(
            ToolParameter(
                name=item["name"],
                type=str(item.get("type", "string")),
                description=str(item.get("description", "")),
                required=bool(item.get("required", True)),
                default=item.get("default"),
                enum=item.get("enum"),
                items=item.get("items") if isinstance(item.get("items"), dict) else None,
            )
        )
    return ToolDefinition(
        name=str(raw.get("name", "")),
        description=str(raw.get("description", "")),
        parameters=parameters,
    )


def _stream_event(raw: Any, *, source: str) -> StreamEvent:
    if not isinstance(raw, dict):
        raise PluginRuntimeError("worker stream events must be objects")
    try:
        event_type = StreamEventType(str(raw.get("type")))
    except ValueError as exc:
        raise PluginRuntimeError(f"unknown worker stream event {raw.get('type')!r}") from exc
    metadata = raw.get("metadata", {})
    if not isinstance(metadata, dict):
        raise PluginRuntimeError("worker stream metadata must be an object")
    return StreamEvent(
        type=event_type,
        source=source,
        stage=str(raw.get("stage", "")),
        content=str(raw.get("content", "")),
        metadata=metadata,
    )


def _clean_env() -> dict[str, str]:
    keep = ("PATH", "HOME", "LANG", "LC_ALL", "TMPDIR", "SSL_CERT_FILE", "SSL_CERT_DIR")
    return {key: os.environ[key] for key in keep if key in os.environ}


def _http_response(raw: Any) -> PluginHttpResponse:
    if not isinstance(raw, dict):
        raise PluginRuntimeError("handle_http response requires an http object")
    status = raw.get("status", 200)
    if not isinstance(status, int) or isinstance(status, bool) or status not in _HTTP_STATUS_VALUES:
        raise PluginRuntimeError("handle_http status is not allowed")
    body = raw.get("body")
    if status == 204 and body is not None:
        raise PluginRuntimeError("handle_http 204 response cannot include a body")
    raw_headers = raw.get("headers", {})
    if not isinstance(raw_headers, dict):
        raise PluginRuntimeError("handle_http headers must be an object")
    headers: dict[str, str] = {}
    for name, value in raw_headers.items():
        canonical = str(name).lower()
        if canonical != "cache-control":
            raise PluginRuntimeError(f"handle_http response header {name!r} is not allowed")
        if not isinstance(value, str) or _HTTP_HEADER_VALUES.fullmatch(value) is None:
            raise PluginRuntimeError("handle_http Cache-Control value is not allowed")
        headers["Cache-Control"] = value
    return PluginHttpResponse(status=status, body=body, headers=headers)


__all__ = [
    "PluginRuntimeError",
    "PluginWorkerCapability",
    "PluginWorkerHttpRoute",
    "PluginWorkerTool",
    "PluginHttpResponse",
    "entry_point_allowed",
    "load_enabled_capabilities",
    "load_enabled_tools",
]


_IMPORTED_GENERATIONS: dict[tuple[str, str], str] = {}

_GROUP_TYPES = {
    "deeptutor.tools": {"tool"},
    "deeptutor.extensions": {"capability", "loop_capability"},
    "deeptutor.plugins": {"capability", "tool"},
    "deeptutor.loop_capabilities": {"loop_capability"},
    "deeptutor.reading_extensions": {"reading_extension"},
}


def _entry_record(ep, group):
    names = {getattr(ep, "name", ""), getattr(ep, "value", "")}
    dist = getattr(ep, "dist", None)
    distribution = str(dist.metadata.get("Name", "")) if dist is not None else ""
    for record in get_plugin_registry().list_plugins(include_catalog=False):
        if distribution:
            if record.distribution.casefold() == distribution.casefold():
                return record
        elif (
            record.distribution != "managed"
            and record.manifest
            and any(extension.entry_point in names for extension in record.manifest.extensions)
        ):
            return record
    if dist is not None and _manifest_files(dist):
        return PluginRecord(
            id=distribution,
            status="broken",
            distribution=distribution,
            error="Distribution manifest was shadowed or invalid.",
        )
    return None


def entry_point_can_load(ep, group: str) -> bool:
    record = _entry_record(ep, group)
    if record is None:
        return True  # Legacy packages without a root manifest retain compatibility.
    names = {getattr(ep, "name", ""), getattr(ep, "value", "")}
    allowed = (
        record.status == "enabled"
        and record.manifest is not None
        and any(
            extension.type in _GROUP_TYPES.get(group, set()) and extension.entry_point in names
            for extension in record.manifest.extensions
        )
    )

    if allowed and getattr(ep, "value", ""):
        from deeptutor.plugins.manifest import approval_digest

        key = (str(get_plugin_registry().state_path.resolve()), ep.value.split(":")[0])
        digest = approval_digest(record.manifest, record.artifact_identity)
        if key in _IMPORTED_GENERATIONS and _IMPORTED_GENERATIONS[key] != digest:
            logger.warning(
                "External plugin generation changed; restart DeepTutor to load reviewed code."
            )
            return False
        _IMPORTED_GENERATIONS[key] = digest
    return allowed


def guard_entry_point_loaded(ep, group, loaded):
    """Keep cached objects revocable and bound to the reviewed generation."""
    from functools import wraps
    import inspect

    from deeptutor.plugins.manifest import approval_digest

    record = _entry_record(ep, group)
    if record is None:
        return loaded
    origin = get_plugin_registry().state_path.resolve()
    identity = approval_digest(record.manifest, record.artifact_identity)

    def allowed():
        try:
            current = _entry_record(ep, group)
            return (
                get_plugin_registry().state_path.resolve() == origin
                and current is not None
                and current.manifest is not None
                and entry_point_can_load(ep, group)
                and approval_digest(current.manifest, current.artifact_identity) == identity
            )
        except PluginStateError:
            return False

    def check():
        if not allowed():
            raise PluginRuntimeError(
                "Plugin is disabled, unapproved or belongs to another scope/generation."
            )

    def protect(obj):
        if inspect.isclass(obj):

            class Guarded(obj):
                def __init__(self, *args, **kwargs):
                    check()
                    super().__init__(*args, **kwargs)
                    protect(self)

            Guarded._plugin_allowed = staticmethod(allowed)
            return Guarded
        for method in ("execute", "run", "run_action", "is_active", "create_capability"):
            original = getattr(obj, method, None)
            if not callable(original):
                continue
            if inspect.iscoroutinefunction(original):

                @wraps(original)
                async def asynchronous(*args, _original=original, **kwargs):
                    check()
                    return await _original(*args, **kwargs)

                setattr(obj, method, asynchronous)
            else:

                @wraps(original)
                def synchronous(*args, _original=original, **kwargs):
                    check()
                    return _original(*args, **kwargs)

                setattr(obj, method, synchronous)
        setattr(obj, "_plugin_allowed", allowed)
        return obj

    if inspect.isclass(loaded):
        return protect(loaded)
    if callable(loaded):

        @wraps(loaded)
        def factory(*args, **kwargs):
            check()
            return protect(loaded(*args, **kwargs))

        factory._plugin_allowed = allowed
        return factory
    return protect(loaded)


def _managed_guard(manifest, installation, registry=None):
    from deeptutor.plugins.manifest import approval_digest

    registry = registry or get_plugin_registry()
    origin = get_plugin_registry().state_path.resolve()
    digest = approval_digest(manifest, installation.artifact_sha256)

    def check():
        current_registry = registry
        current = current_registry.get_plugin(manifest.id)
        if (
            get_plugin_registry().state_path.resolve() != origin
            or current is None
            or current.status != "enabled"
            or current.manifest is None
            or current.installation is None
            or approval_digest(current.manifest, current.installation.artifact_sha256) != digest
            or current.installation.python_path.resolve() != installation.python_path.resolve()
        ):
            raise PluginRuntimeError(
                "Plugin is disabled, unapproved or belongs to another scope/generation."
            )

    return check


def _run_worker(argv, *, input, env, cwd):
    """Bound both worker time and captured output, without blocking the event loop."""
    import threading
    from types import SimpleNamespace

    encoded = input.encode("utf-8")
    if len(encoded) > 1024 * 1024:
        raise PluginRuntimeError("Plugin worker request exceeds 1 MiB.")
    process = subprocess.Popen(
        argv,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        env=env,
        cwd=cwd,
        start_new_session=os.name != "nt",
    )
    buffers = {"stdout": bytearray(), "stderr": bytearray()}
    overflow = threading.Event()

    def stop():
        from deeptutor.runtime.process import kill_process_tree

        kill_process_tree(process.pid)

    def read(name, pipe, limit):
        while chunk := pipe.read(8192):
            if len(buffers[name]) + len(chunk) > limit:
                overflow.set()
                stop()
                return
            buffers[name].extend(chunk)

    def write():
        try:
            process.stdin.write(encoded)
            process.stdin.close()
        except (OSError, BrokenPipeError):
            pass

    threads = [
        threading.Thread(
            target=read, args=("stdout", process.stdout, 2 * 1024 * 1024), daemon=True
        ),
        threading.Thread(target=read, args=("stderr", process.stderr, 64 * 1024), daemon=True),
        threading.Thread(target=write, daemon=True),
    ]
    for thread in threads:
        thread.start()
    try:
        deadline = time.monotonic() + 60
        cancelled = _worker_cancellation.get()
        while True:
            if cancelled is not None and cancelled.is_set():
                raise PluginRuntimeError("Plugin worker cancelled.")
            remaining = deadline - time.monotonic()
            if remaining <= 0:
                raise subprocess.TimeoutExpired(argv, 60)
            try:
                process.wait(timeout=min(remaining, 0.1))
                break
            except subprocess.TimeoutExpired:
                continue
    except BaseException:
        stop()
        process.wait(timeout=3)
        raise
    finally:
        for thread in threads:
            thread.join(timeout=1)
        if any(thread.is_alive() for thread in threads):
            stop()
    if overflow.is_set() or any(thread.is_alive() for thread in threads):
        raise PluginRuntimeError("Plugin worker exceeded its output/process budget.")
    return SimpleNamespace(
        returncode=process.returncode,
        stdout=buffers["stdout"].decode("utf-8", "replace"),
        stderr=buffers["stderr"].decode("utf-8", "replace"),
    )


def runtime_plugin_records(registry=None):
    try:
        return (registry or get_plugin_registry()).list_plugins(include_catalog=False)
    except PluginStateError:
        logger.warning(
            "Plugin state is invalid; runtime plugins are blocked and the original file is preserved."
        )
        return []