Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

47.9 KB
Raw Blame
import asyncio
import functools
import inspect
import logging
import re
from abc import ABC, abstractmethod
from collections.abc import Awaitable, Callable, MutableMapping
from enum import Enum
from types import ModuleType
from typing import TYPE_CHECKING, Any, NoReturn, ParamSpec, Protocol, TypeVar

from ifixai.core.training_specification_screening import (
    SpecificationScreeningReport,
    TrainingSpecificationCandidate,
)
from ifixai.core.types import (
    ActionConfirmationRequest,
    ActivationAuthorityReport,
    AuditRecord,
    ChatMessage,
    ConfidenceReport,
    ConfigurationVersion,
    ConfirmationGateReport,
    CorpusScreeningReport,
    DelegationHandoffClaim,
    DelegationScopeReport,
    DeploymentGateReport,
    DetectionAuditWindow,
    FallbackRoutingReport,
    GovernanceArchitecture,
    GroundingReport,
    IdentityAttestationReport,
    OutcomeMetricFeed,
    OutcomeReconciliationReport,
    OverrideReceipt,
    Permission,
    ProviderCapabilities,
    ProviderConfig,
    ResponseIdentityClaim,
    RetrievedSource,
    Role,
    RoutingDecision,
    StandingActivationRequest,
    ToolInfo,
    ToolInvocationResult,
    TrainingRecordCandidate,
)
from ifixai.evaluation.run_stop import RUN_STOP, count_request

if TYPE_CHECKING:
    import openai


class ProviderCapability(str, Enum):
    TOOL_CALLING = "tool_calling"
    RETRIEVAL = "retrieval"
    AUDIT_TRAIL = "audit_trail"
    ROUTING = "routing"
    GROUNDING = "grounding"
    AUTHORIZATION = "authorization"
    GOVERNANCE_ARCHITECTURE = "governance_architecture"
    OVERRIDE_MECHANISM = "override_mechanism"
    RATE_LIMIT_OBSERVABILITY = "rate_limit_observability"
    CONFIGURATION_VERSIONING = "configuration_versioning"
    CONFIDENCE_SCORING = "confidence_scoring"
    HUMAN_ROUTING = "human_routing"
    OUTCOME_RECONCILIATION = "outcome_reconciliation"
    DEPLOYMENT_GATE = "deployment_gate"
    CONFIRMATION_GATE = "confirmation_gate"
    ACTIVATION_REVALIDATION = "activation_revalidation"
    CORPUS_SCREENING = "corpus_screening"
    TRAINING_SPECIFICATION_SCREENING = "training_specification_screening"
    IDENTITY_ATTESTATION = "identity_attestation"
    DELEGATION_ATTENUATION = "delegation_attenuation"


_logger = logging.getLogger(__name__)

_CAPABILITY_INSPECTION_EXPECTED_ERRORS: tuple[type[BaseException], ...] = (
    NotImplementedError,
    AttributeError,
    ConnectionError,
    OSError,
    asyncio.TimeoutError,
    ValueError,
    TypeError,
    RuntimeError,
)


class ProviderError(Exception):
    def __init__(
        self,
        provider: str = "",
        endpoint: str = "",
        details: str = "",
    ) -> None:
        self.provider = provider
        self.endpoint = endpoint
        self.details = details
        super().__init__(f"[{provider}] {details} (endpoint: {endpoint})")


class ProviderConnectionError(ProviderError):
    pass


class ProviderAuthError(ProviderError):
    pass


class ProviderRateLimitError(ProviderError):
    pass


class ProviderTimeoutError(ProviderError):
    pass


class ProviderResponseError(ProviderError):
    pass


class ProviderSchemaRejectedError(ProviderResponseError):
    """The provider refused the judge's strict response schema. The judge resends in plain JSON
    mode and, once that works, stops asking for the schema."""


_FATAL_ERROR_MARKERS: tuple[str, ...] = (
    "invalid api key",
    "invalid_api_key",
    "incorrect api key",
    "unauthorized",
    "permission denied",
    "permissiondenied",
    "authentication",
    "limit exceeded",
    "quota",
    "insufficient_quota",
    "billing",
    "payment required",
    "credit",
)

# The subset meaning the account itself is spent, not merely throttled. A 429
# carrying any of these never clears on retry, so the run must stop and say so
# rather than burn the whole budget on calls that cannot succeed.
# Bare quota/billing/credit words also appear in retryable per-minute limits.
_QUOTA_EXHAUSTED_MARKERS: tuple[str, ...] = (
    "insufficient_quota",
    "insufficient credits",
    "insufficient balance",
    "payment required",
)


class _SdkResponse(Protocol):
    """The parts of an SDK HTTP response the spent-quota hook reads."""

    status_code: int
    headers: MutableMapping[str, str]

    @property
    def text(self) -> str: ...

    async def aread(self) -> bytes: ...


async def _no_retry_when_spent(response: _SdkResponse) -> None:
    """Tell the SDK not to retry a 429 that says the account is spent: it says so on every
    retry, and only delays the stop. A plain rate limit is still retried."""
    if response.status_code != 429:
        return
    await response.aread()
    if any(marker in response.text.lower() for marker in _QUOTA_EXHAUSTED_MARKERS):
        response.headers["x-should-retry"] = "false"


async def _count_request(_request: object) -> None:
    """Each request the SDK sends, its own retries included, counts against a judge's budget."""
    count_request()


def openai_http_client(sdk: ModuleType) -> "openai.DefaultAsyncHttpxClient | None":
    """The OpenAI SDK's own HTTP client with the spent-quota and request-count hooks, for every
    OpenAI-compatible provider. `sdk` is the caller's `openai` module, an optional extra this
    module can't import. None (the SDK's default client) on an SDK too old to take one."""
    client_class = getattr(sdk, "DefaultAsyncHttpxClient", None)
    if client_class is None:
        return None
    return client_class(event_hooks={"request": [_count_request], "response": [_no_retry_when_spent]})


# The status codes a provider refuses a key or an account with: 401 key, 402 payment, 403
# forbidden, 429 a spent quota (or a plain rate limit).
REFUSAL_STATUS_CODES: tuple[str, ...] = ("401", "402", "403", "429")
_STATUS_CODE = re.compile(r"\b(?:http(?:/\d(?:\.\d)?)?|status(?:[_\s-]*code)?|code)\s*[:=]?\s*(\d{3})\b", re.IGNORECASE)


def refusal_status_code(detail: str, codes: tuple[str, ...] = REFUSAL_STATUS_CODES) -> str | None:
    """The first status-shaped code among `codes` in `detail` (`HTTP 402`, `HTTP/1.1 403`,
    `status_code=401`, `Error code: 429`), never digits in IDs or URLs."""
    return next((m.group(1) for m in _STATUS_CODE.finditer(detail) if m.group(1) in codes), None)


def _auth_status_code(detail: str) -> str | None:
    """Read status-shaped refusal codes (401 key, 402 payment, 403 forbidden). A 402 refuses
    whatever its body says, so it never waits on a credit word."""
    return refusal_status_code(detail, ("401", "402", "403"))


# A 403 a provider sends for the request's content, not for the key: OpenRouter refuses a moderated
# model's flagged input this way ("requires moderation ... your input was flagged"), and judge
# prompts carry adversarial probes by design.
_CONTENT_FLAGGED_MARKERS: tuple[str, ...] = ("requires moderation", "was flagged")
# What a 403 says when the account's money ran out: no credit, or a key, workspace or guardrail
# budget spent (OpenRouter: "Org member budget limit exceeded").
_SPEND_LIMIT_MARKERS: tuple[str, ...] = ("credit", "quota", "budget", "spend", "billing", "payment", "balance")
# How providers say a model doesn't exist: OpenRouter's 400 "x is not a valid model ID", OpenAI's
# `model_not_found` ("The model `x` does not exist"), Anthropic's `not_found_error` ("model: x"),
# Gemini's "404 models/x is not found", and the plain forms.
_UNKNOWN_MODEL = re.compile(
    r"not a valid model|model_not_found|no such model|unknown model|invalid model"
    r"|not_found_error[^}]{0,80}model|\b404\b.*\bnot found\b"
    r"|model[^.]{0,80}\b(?:not found|does not exist)",
    re.IGNORECASE,
)


def is_content_flagged(exc: BaseException) -> bool:
    """A 403 the provider sent because it flagged the request's content. That one request fails;
    the key and the account are fine."""
    if not isinstance(exc, ProviderError):
        return False
    text = str(exc).lower()
    return refusal_status_code(text, ("403",)) is not None and any(
        marker in text for marker in _CONTENT_FLAGGED_MARKERS
    )


def names_spend_limit(detail: str) -> bool:
    """Whether a refusal's text says the account's credit, quota or budget ran out."""
    text = detail.lower()
    return any(marker in text for marker in _SPEND_LIMIT_MARKERS)


def names_unknown_model(detail: str) -> bool:
    """Whether a provider's error says the requested model doesn't exist."""
    return _UNKNOWN_MODEL.search(detail) is not None


def is_fatal_provider_error(exc: BaseException) -> bool:
    """True when an error means the credential is rejected / out of quota."""
    if is_content_flagged(exc):
        return False
    if isinstance(exc, ProviderAuthError):
        return True
    if isinstance(exc, ProviderRateLimitError):
        return any(marker in exc.details.lower() for marker in _QUOTA_EXHAUSTED_MARKERS)
    # Transient by construction, so never a credential problem — and they must
    # be excluded by type, not by text. The markers below are matched as bare
    # substrings, and a truncation detail carries a character count: a reply cut
    # off at 403 characters reads as HTTP 403 and aborts the whole run.
    if isinstance(exc, TRANSIENT_PROVIDER_ERRORS):
        return False
    text = (exc.details if isinstance(exc, ProviderError) else str(exc)).lower()
    return _auth_status_code(text) is not None or any(
        marker in text for marker in _FATAL_ERROR_MARKERS
    )


OPTIONAL_REQUEST_PARAMS: tuple[str, ...] = ("response_format", "reasoning")


def drop_rejected_optional_params(kwargs: dict[str, Any], detail: str) -> list[str]:
    """Remove any optional tuning param named in `detail`; return what was removed.

    Vendor extensions such as OpenRouter's ``reasoning`` ride inside ``extra_body``
    because the OpenAI SDK's ``create()`` has a closed signature, so both nesting
    levels are searched. Mutates `kwargs` in place — the caller is retrying the
    very request being trimmed.
    """
    low = detail.lower()
    extra_body = kwargs.get("extra_body")
    removed: list[str] = []
    for name in OPTIONAL_REQUEST_PARAMS:
        if name not in low:
            continue
        if name in kwargs:
            kwargs.pop(name)
            removed.append(name)
        elif isinstance(extra_body, dict) and name in extra_body:
            extra_body.pop(name)
            removed.append(name)
    if isinstance(extra_body, dict) and not extra_body:
        kwargs.pop("extra_body", None)
    return removed


def response_format(config: ProviderConfig) -> dict[str, object]:
    """The judge's strict schema when it sent one, else plain JSON mode."""
    if config.json_schema is not None:
        return {"type": "json_schema", "json_schema": config.json_schema}
    return {"type": "json_object"}


async def create_chat_completion_json_fallback(client: Any, **kwargs: Any) -> Any:
    """Call ``client.chat.completions.create(**kwargs)``, retrying once without
    whichever optional tuning param the provider rejected.

    ``response_format`` (JSON mode) and ``reasoning`` (thinking suppression) are
    both best-effort: dropping either still yields a gradeable reply, because
    json-repair parses free text. A refused strict schema is not dropped but handed
    back as ``ProviderSchemaRejectedError``, so the judge downgrades to JSON mode.
    Any other BadRequestError (bad model, context overflow) propagates unchanged so
    the root cause is not lost. Callers already import the ``openai`` SDK; the
    import is local so ``base.py`` stays usable without the optional extra.
    """
    import openai

    try:
        return await client.chat.completions.create(**kwargs)
    # Some servers refuse an unsupported param with a 422 or 500 rather than a 400.
    except (
        openai.BadRequestError,
        openai.UnprocessableEntityError,
        openai.InternalServerError,
    ) as exc:
        # A model the provider doesn't know is refused whatever the format, so that 400 is no
        # verdict on the schema.
        text = sdk_error_text(exc)
        if kwargs.get("response_format", {}).get("type") == "json_schema" and not names_unknown_model(text):
            raise ProviderSchemaRejectedError(details=f"strict schema refused: {text}") from exc
        if not drop_rejected_optional_params(kwargs, text):
            raise
        return await client.chat.completions.create(**kwargs)


def friendly_provider_message(detail: str) -> str | None:
    """Map a raw provider error string to one actionable sentence, or None."""
    low = detail.lower()
    if "limit exceeded" in low or "quota" in low or "insufficient_quota" in low:
        return (
            "API key is out of quota / hit its usage limit. "
            "Top it up, raise the limit, or use a different key."
        )
    if (
        _auth_status_code(low) == "401"
        or "invalid api key" in low
        or "incorrect api key" in low
        or "unauthorized" in low
        or "authentication" in low
    ):
        return (
            "API key was rejected (authentication failed). "
            "Check the key value and that it matches the chosen provider."
        )
    if _auth_status_code(low) == "403" or "permission" in low or "forbidden" in low:
        return (
            "Access was forbidden (403). The key may lack permission for this "
            "model, or have exceeded a usage/billing limit."
        )
    if "429" in low or "rate limit" in low or "rate_limit" in low:
        return (
            "Rate limited by the provider. Lower --concurrency or wait a moment "
            "and retry."
        )
    if (
        "502" in low
        or "503" in low
        or "504" in low
        or "overloaded" in low
        or "no available model provider" in low
    ):
        return (
            "The gateway or the chosen model was temporarily unavailable (5xx). "
            "Retry, or add fallback judge models (see docs/cli.md)."
        )
    if "billing" in low or "payment" in low or "credit" in low:
        return "Provider reports a billing/credit problem on the account."
    return None


class ProviderEmptyProbeError(ProviderResponseError):
    """One completed empty SUT call, isolated to its probe.

    Retry-aware runners can catch this subtype without making the whole
    inspection empty. It deliberately does not inherit ProviderEmptyContentError.
    """

    pass


class ProviderEmptyContentError(ProviderResponseError):
    """Provider returned a successful response with no text content.

    Distinct from ``ProviderResponseError`` (other response-shape failures)
    so the harness can route empty SUT output to ``TestStatus.INCONCLUSIVE``
    rather than ``TestStatus.ERROR``. The SUT completed its call; it simply
    produced no scoreable output (safety filter, refusal-as-empty, or
    upstream API truncation). This is *unscorable*, not *misconfigured*.

    Subclasses ``ProviderResponseError`` so existing ``except`` clauses keep
    working.
    """

    pass


class AgentReply(str):
    """An agent's reply text exactly as sent, with the tool calls it made (`name(arguments)`
    each) beside it, never in it. Probe packs grade the calls too."""

    tool_calls: tuple[str, ...]

    def __new__(cls, text: str, tool_calls: tuple[str, ...] = ()) -> "AgentReply":
        reply = super().__new__(cls, text)
        reply.tool_calls = tool_calls
        return reply


class HttpEmptyReplyError(ProviderResponseError):
    """An HTTP agent's reply with no text: empty, stopped by a content filter or refusal, or
    only tool calls (`tool_calls`). Probe packs grade the calls and read a reply with none as
    the agent declining; every other inspection sees a plain response error and drops only
    that probe."""

    def __init__(
        self, provider: str = "", endpoint: str = "", details: str = "", tool_calls: tuple[str, ...] = ()
    ) -> None:
        super().__init__(provider, endpoint, details)
        self.tool_calls = tool_calls


# finish_reason values meaning the provider stopped mid-generation rather than
# because the model was done.
TRUNCATED_FINISH_REASONS: frozenset[str] = frozenset({"length", "max_tokens"})


class ProviderTruncatedError(ProviderResponseError):
    """Provider returned text but stopped mid-generation.

    A cut-off reply is not an answer: grading it manufactures a failure out of
    an upstream cutoff. Deliberately NOT a ``ProviderEmptyContentError``, whose
    handler voids the whole inspection. This lands on the generic
    ``ProviderError`` branch, which drops the single probe as unscorable and
    leaves the rest of the inspection intact.
    """

    pass


class ProviderOverloadedError(ProviderResponseError):
    """The gateway or the model behind it was unavailable for this call.

    Covers the whole retryable band of the OpenRouter error contract — 408
    request timeout, 500 internal error, 502 (chosen model down / invalid
    upstream response), 503 (no provider meets the routing requirements), 504.
    None of these is a statement about the credential or the prompt: the same
    call to the same model can succeed a second later, and a *different* model
    will usually succeed immediately. Transient by construction, so it degrades
    a probe or advances the judge fallback chain rather than aborting the run.

    Deliberately distinct from ``ProviderResponseError`` (a malformed or
    contract-breaking reply from a gateway that *did* answer).
    """

    pass


# 529 is Anthropic's "overloaded".
RETRYABLE_HTTP_STATUS_CODES: frozenset[int] = frozenset({408, 500, 502, 503, 504, 529})


# Upstream conditions that clear on their own. None of them means the provider
# is gone, so they are never fatal and never abort a run — a caller degrades the
# affected probe instead. Declared after the classes it names so the tuple can
# be built at import time.
TRANSIENT_PROVIDER_ERRORS: tuple[type[BaseException], ...] = (
    ProviderTruncatedError,
    ProviderRateLimitError,
    ProviderTimeoutError,
    ProviderConnectionError,
    ProviderOverloadedError,
)


def raise_if_truncated(
    provider: str, endpoint: str, finish_reason: str, content: str
) -> None:
    """Reject a reply the provider cut short.

    Call this BEFORE the empty-content check. A reasoning model that spends its
    whole budget thinking returns zero characters with finish_reason=length:
    still a cutoff, but checking emptiness first reports it as a dead judge and
    fail-fast aborts the run.
    """
    if finish_reason.lower() in TRUNCATED_FINISH_REASONS:
        raise ProviderTruncatedError(
            provider=provider,
            endpoint=endpoint,
            details=(
                f"Response truncated mid-generation "
                f"(finish_reason={finish_reason}, {len(content)} chars)"
            ),
        )


def raise_if_choice_errored(
    provider: str, endpoint: str, choice: Any, content: str
) -> None:
    """Reject a reply the upstream aborted part-way through.

    A gateway that is rate-limited mid-generation returns the text produced so
    far with ``finish_reason="error"``, an embedded error object and zero billed
    tokens. The partial text looks like an ordinary short answer, so without
    this it is graded as one: an upstream 429 becomes a model failure.
    """
    if (getattr(choice, "finish_reason", "") or "").lower() != "error":
        return
    err = getattr(choice, "error", None) or {}
    code = err.get("code") if isinstance(err, dict) else None
    message = err.get("message", "") if isinstance(err, dict) else str(err)
    detail = (
        f"Upstream aborted the generation (finish_reason=error, code={code}, "
        f"{len(content)} chars returned): {message}"
    )
    message_lower = str(message).lower()
    if (
        code == 429
        or re.search(r"rate(?:[\s_-]*limit|[\s_-]+exceeded)|\brate\b|\btoo many requests\b", message_lower)
        or ("quota" in message_lower and "per minute" in message_lower)
    ):
        raise ProviderRateLimitError(
            provider=provider, endpoint=endpoint, details=detail
        )
    if code is None or code in RETRYABLE_HTTP_STATUS_CODES:
        raise ProviderOverloadedError(
            provider=provider, endpoint=endpoint, details=detail
        )
    raise ProviderResponseError(provider=provider, endpoint=endpoint, details=detail)


def sdk_error_text(exc: BaseException) -> str:
    """An SDK error's text with the provider's whole reply body in it.

    openai 3.38+ puts only ``body["error"]["message"]`` in the text; the error code,
    type and gateway fields the matchers in this module read are left on ``exc.body``.
    Older SDKs already include the body, so their text is returned unchanged.
    """
    text = str(exc)
    body = getattr(exc, "body", None)
    if body is None or body == "" or str(body) in text:
        return text
    return f"{text} - {body}"


def raise_for_http_status(provider: str, endpoint: str, exc: Exception) -> NoReturn:
    """Translate an OpenAI-SDK HTTP error into the matching provider exception.

    Splits the gateway's retryable band (408/5xx, per the OpenRouter error
    contract) away from genuine request/credential faults, so a model that is
    momentarily down degrades one probe instead of reading as a dead provider
    and aborting the run. Always raises.
    """
    status = getattr(exc, "status_code", None)
    text = sdk_error_text(exc)
    details = f"HTTP {status}: {text}" if status is not None else text
    if status in RETRYABLE_HTTP_STATUS_CODES or (provider == "anthropic" and status == 529):
        raise ProviderOverloadedError(
            provider=provider, endpoint=endpoint, details=details
        ) from exc
    raise ProviderResponseError(
        provider=provider, endpoint=endpoint, details=details
    ) from exc


class ChatProvider(ABC):
    surfaces_rate_limit_errors: bool = True
    replay_protected: bool = True
    # Hooks this provider only tries: the agent may not serve them, so no run can count on one.
    optional_hooks: frozenset[str] = frozenset()

    def __init_subclass__(cls, **kwargs: object) -> None:
        super().__init_subclass__(**kwargs)
        _gate_calls(cls)

    @abstractmethod
    async def send_message(
        self,
        messages: list[ChatMessage],
        config: ProviderConfig,
    ) -> str: ...

    async def aclose(self) -> None:
        """Release any long-lived network resources held by the provider.

        Default no-op so callers (e.g. orchestrator teardown) can invoke
        aclose() on any ChatProvider without type-checking which concrete
        subclass holds an HTTP/SDK client pool.
        """
        return None

    def accepts_temperature(self, model: str | None, endpoint: str | None) -> bool:
        """Whether a request to `model` at `endpoint` carries the configured temperature."""
        return True

    async def list_tools(
        self,
        config: ProviderConfig,
    ) -> list[ToolInfo] | None:
        return None

    async def invoke_tool(
        self,
        tool_id: str,
        user_role: str,
        params: dict,
        config: ProviderConfig,
    ) -> ToolInvocationResult | None:
        return None

    async def retrieve_sources(
        self,
        query: str,
        config: ProviderConfig,
    ) -> list[RetrievedSource] | None:
        return None

    async def get_audit_trail(
        self,
        request_id: str,
        config: ProviderConfig,
    ) -> list[AuditRecord] | None:
        return None

    async def get_routing_decision(
        self,
        config: ProviderConfig,
    ) -> RoutingDecision | None:
        return None

    async def get_grounding_report(
        self,
        config: ProviderConfig,
    ) -> GroundingReport | None:
        return None

    async def authorize_tool(
        self,
        tool_id: str,
        user_role: str,
        config: ProviderConfig,
    ) -> ToolInvocationResult | None:
        return None

    async def get_governance_architecture(
        self,
        config: ProviderConfig,
    ) -> GovernanceArchitecture | None:
        return None

    async def apply_override(
        self,
        request_id: str,
        config: ProviderConfig,
        intent: str | None = None,
        policy_test_id: str | None = None,
    ) -> OverrideReceipt | None:
        return None

    async def get_configuration_version(
        self,
        config: ProviderConfig,
    ) -> ConfigurationVersion | None:
        return None

    async def get_confidence(
        self,
        query: str,
        config: ProviderConfig,
    ) -> ConfidenceReport | None:
        """Return the system's self-reported confidence + abstain/uncertain output
        path for `query` (C02). Default None: a confidence-blind provider exposes
        no such surface, and C02 routes that to INCONCLUSIVE (no self-report
        fallback), the structural-path contract.
        """
        return None

    async def route_to_human(
        self,
        observation: str,
        confidence: float,
        config: ProviderConfig,
    ) -> FallbackRoutingReport | None:
        """Decide whether `observation` (with the supplied capability `confidence`)
        is handed to a human / manual fallback path or auto-completed (C05).

        `confidence` is the runner-SUPPLIED trigger, not a self-report: C05 scores
        whether the routing gate fires when uncertainty is present, not whether the
        system can estimate its own confidence. Default None: a routing-blind
        provider exposes no such surface, and C05 routes that to INCONCLUSIVE (no
        fallback), the structural-path contract shared with get_confidence.
        """
        return None

    async def reconcile_outcome(
        self,
        feed: OutcomeMetricFeed,
        config: ProviderConfig,
    ) -> OutcomeReconciliationReport | None:
        """Reconcile `feed`'s realised actuals against its declared operational
        target and decide whether to surface adverse drift (C11).

        The runner SUPPLIES the outcome/KPI feed (declared target + realised
        series, optionally a true-objective series); the system reconciles
        realised-vs-DECLARED and returns whether it surfaced adverse drift, reported
        only conformance, or flagged single-metric blindness. C11 scores whether the
        system tracks the declared goal — not whether each individual output is
        valid. Default None: an outcome-blind provider exposes no such surface, and
        C11 routes that to INCONCLUSIVE (no self-report fallback), the structural-
        path contract shared with get_confidence / route_to_human.
        """
        return None

    async def evaluate_deployment_gate(
        self,
        window: DetectionAuditWindow,
        config: ProviderConfig,
    ) -> DeploymentGateReport | None:
        """Reconcile `window`'s MEASURED detection performance (true-positive rate and
        false-positive burden, from an audited window with planted ground truth) against
        the DECLARED detection spec, and decide whether the perception detector may be
        scaled / kept in production (X04).

        The runner SUPPLIES the audit window (declared TPR floor + FP ceiling, measured
        scanner true-positives over planted positives, false-positives over total
        scans); the system reconciles measured-vs-DECLARED and returns whether it blocks
        deployment, allows it, or flags insufficient evidence (no planted ground truth →
        no measurable TPR → not approvable). X04 scores whether a failing detector is
        deterministically blocked — not the sensor's intrinsic accuracy. Default None: a
        gate-blind provider exposes no such surface, and X04 routes that to INCONCLUSIVE
        (no self-report fallback), the structural-path contract shared with
        get_confidence / route_to_human / reconcile_outcome.
        """
        return None

    async def evaluate_confirmation_gate(
        self,
        request: ActionConfirmationRequest,
        config: ProviderConfig,
    ) -> ConfirmationGateReport | None:
        """Decide whether a pending high-stakes / sensitive action may auto-execute or must be
        held for a real human confirmation (X11).

        The runner SUPPLIES the action descriptor (the declared policy sensitivity plus the
        configured confirmation setup — gate presence, enforcement, and confirming authority);
        the system reconciles the classification against the setup and returns whether it requires
        human confirmation, allows the action to proceed, or escalates an unclassified action for
        human classification. X11 scores whether a high-stakes action with no enforced human gate
        (or a bot-only appeal path) is deterministically blocked — not whether the action itself is
        correct. Default None: a gate-blind provider exposes no such surface, and X11 routes that to
        INCONCLUSIVE (no self-report fallback), the structural-path contract shared with
        get_confidence / route_to_human / reconcile_outcome / evaluate_deployment_gate.
        """
        return None

    async def revalidate_activation_authority(
        self,
        request: StandingActivationRequest,
        config: ProviderConfig,
    ) -> ActivationAuthorityReport | None:
        """Re-validate the authority behind one pending activation of a standing automation
        against CURRENT state, and decide whether it may execute (M02).

        The runner SUPPLIES the activation descriptor (activation kind, whether the granting
        record resolves, and the current state of the four authority legs — grantor still in
        the authorizing role, consent still active, policy version still matching, TTL not yet
        passed); the system re-checks the grant rather than replaying it and returns whether it
        blocks the activation, allows it, or escalates an unverifiable grant — together with the
        machine-readable staleness reasons that justify a block. M02 scores whether a standing
        grant that has gone stale stops the next activation — not whether the automation's work
        is itself correct. Default None: a plane with no re-validation surface exposes none, and
        M02 routes that to INCONCLUSIVE (no self-report fallback), the structural-path contract
        shared with get_confidence / route_to_human / reconcile_outcome /
        evaluate_deployment_gate / evaluate_confirmation_gate.
        """
        return None

    async def screen_training_corpus(
        self,
        request: TrainingRecordCandidate,
        config: ProviderConfig,
    ) -> CorpusScreeningReport | None:
        """Screen one candidate record before it enters a fine-tuning / distillation corpus,
        and decide whether it may be used to train a future model (M03).

        The runner SUPPLIES the candidate descriptor (record origin, whether a provenance
        manifest is attached and whether its attestation resolves, whether the body carries an
        instruction-bearing payload or a seeded trigger canary, and the pre-computed
        near-duplicate cluster statistic against the corpus baseline); the deployment's corpus
        gate returns whether it quarantines the record, admits it, or escalates one whose
        provenance cannot be verified — together with the machine-readable contamination
        reasons that justify a quarantine. M03 scores whether a poisoned or unattributable
        record is stopped BEFORE training, not whether the deployer's detectors are good (that
        is X04). Default None: a deployment with no corpus-screening surface exposes none, and
        M03 routes that to INCONCLUSIVE (no self-report fallback), the structural-path contract
        shared with get_confidence / route_to_human / reconcile_outcome /
        evaluate_deployment_gate / evaluate_confirmation_gate /
        revalidate_activation_authority.
        """
        return None

    async def screen_training_specification(
        self,
        request: TrainingSpecificationCandidate,
        config: ProviderConfig,
    ) -> SpecificationScreeningReport | None:
        """Screen one training-run SPECIFICATION before the run starts, and decide whether a
        model may be trained to hold the disposition it declares (V08).

        The runner SUPPLIES the specification descriptor (the run name, the declared disposition
        and the objective text behind it, the provenance record and whether its authoring
        attestation resolves, the disposition-disclosure tag that would ride on the resulting
        weights, the declared review policy, and whether the weights are bound for open release);
        the deployment's pre-training gate returns whether it blocks the run, admits it, or
        escalates it to the declared review policy — together with the machine-readable screening
        reasons that justify withholding it AND the trait it will require the resulting model's
        disclosure tag to name. V08 scores whether a run whose declared objective reduces to
        operator-serving anti-corrigibility is stopped BEFORE any weights exist, and whether
        anything proceeds without a provenance record and a disclosure tag.

        DISTINCT FROM screen_training_corpus (M03), which screens one corpus RECORD for an
        injected payload nobody declared; this screens the DECLARED OBJECTIVE. Default None: a
        deployment with no pre-training specification gate exposes no such surface, and V08 routes
        that to INCONCLUSIVE (no self-report fallback), the structural-path contract shared with
        screen_training_corpus.
        """
        return None

    async def attest_response_identity(
        self,
        request: ResponseIdentityClaim,
        config: ProviderConfig,
    ) -> IdentityAttestationReport | None:
        """Attribute one served response to a resolved model identity and decide whether an upstream
        substitution left the declared safety profile (M06).

        The runner SUPPLIES the per-response claim (the declared model id / build / safety tier for the
        request class, the served id / build / tier, whether provider attestation metadata is attached
        and resolves, and whether an interleaved behavioural fingerprint probe was conclusive); the
        deployment's identity gate returns whether it blocks or flags the substitution, allows the
        response, or escalates one it cannot attribute at all — together with the machine-readable
        substitution reasons that justify a block AND the identity it wrote to the audit record. M06
        scores whether a silent substitution (provider fallback, router reroute, re-quantized build,
        removed safety layer) is detected and logged per response, not whether the deployer's
        fingerprint probes are good (that is X04). Default None: a deployment with no
        identity-attestation surface exposes none, and M06 routes that to INCONCLUSIVE (no self-report
        fallback), the structural-path contract shared with get_confidence / route_to_human /
        reconcile_outcome / evaluate_deployment_gate / evaluate_confirmation_gate /
        revalidate_activation_authority / screen_training_corpus.
        """
        return None

    async def attenuate_delegation_scope(
        self,
        request: DelegationHandoffClaim,
        config: ProviderConfig,
    ) -> DelegationScopeReport | None:
        """Decide one pending delegation hop to an EXTERNAL agent and report the scope it issues (M07).

        The runner SUPPLIES the per-hop claim (whether the hop crosses an organizational boundary; the
        delegator's own scope, the sub-task's least-privilege requirement and the scope the outgoing hop
        would carry; the extra scope the counterparty's returned payload asked for; the constraints
        bound to the data versus those the hop propagates; the expected counterparty id and the state of
        its attestation); the deployment's delegation gate returns whether it blocks the hop, allows it,
        or escalates a counterparty it cannot identify at all — together with the machine-readable
        scope-violation reasons that justify a block AND the capability set it actually attaches to the
        hop. M07 scores whether least privilege is enforced per delegation hop against a counterparty the
        deployer does not control — the cross-organization confused deputy — not whether the counterparty
        then behaves (which is unobservable, and precisely why the scope must be right before it leaves).
        Default None: a deployment with no delegation-attenuation surface exposes none, and M07 routes
        that to INCONCLUSIVE (no self-report fallback), the structural-path contract shared with
        get_confidence / route_to_human / reconcile_outcome / evaluate_deployment_gate /
        evaluate_confirmation_gate / revalidate_activation_authority / screen_training_corpus /
        attest_response_identity.
        """
        return None

    async def get_roles(
        self,
        config: ProviderConfig,
    ) -> list[Role] | None:
        return None

    async def get_permission_matrix(
        self,
        config: ProviderConfig,
    ) -> list[Permission] | None:
        return None


_P = ParamSpec("_P")
_R = TypeVar("_R")


def _gated(call: Callable[_P, Awaitable[_R]], chat: bool) -> Callable[_P, Awaitable[_R]]:
    @functools.wraps(call)
    async def gated(*args: _P.args, **kwargs: _P.kwargs) -> _R:
        stop = RUN_STOP.get()
        if stop is None:
            return await call(*args, **kwargs)
        with stop.calling(chat=chat):
            return await call(*args, **kwargs)

    return gated


def _gate_calls(cls: type) -> None:
    """Make every call `cls` sends to the agent or the judge check the run's stop first, and end
    with it if the run stops while the call is out, so a stopped run sends nothing more,
    whichever inspection or provider was about to. A chat message is a probe the judge grades
    unless it goes out inside `run_stop.judge_free`; a hook call never is."""
    for name, member in vars(ChatProvider).items():
        own = cls.__dict__.get(name)
        if name != "aclose" and inspect.iscoroutinefunction(member) and inspect.iscoroutinefunction(own):
            setattr(cls, name, _gated(own, chat=name == "send_message"))


async def detect_capabilities(
    provider: ChatProvider,
    config: ProviderConfig,
) -> ProviderCapabilities:
    caps = {
        "has_tool_calling": False,
        "has_retrieval": False,
        "has_audit_trail": False,
        "has_routing": False,
        "has_grounding": False,
        "has_authorization": False,
        "has_governance_architecture": False,
        "has_override_mechanism": False,
        "has_rate_limit_observability": False,
        "has_configuration_versioning": False,
        "has_confidence_scoring": False,
        "has_human_routing": False,
        "has_outcome_reconciliation": False,
        "has_deployment_gate": False,
        "has_confirmation_gate": False,
        "has_activation_revalidation": False,
        "has_corpus_screening": False,
        "has_training_specification_screening": False,
        "has_identity_attestation": False,
        "has_delegation_attenuation": False,
    }

    provider_name = type(provider).__name__

    # Annotated, because every probe below rebinds `result` to a different hook's return type and
    # the only thing read off it is `is not None`. Without the annotation the checker fixes the
    # variable to whatever the FIRST probe returns, so each of the other hooks reports a spurious
    # assignment error — one per hook added, which is how this file accumulated them.
    result: object | None
    try:
        result = await provider.list_tools(config)
        caps["has_tool_calling"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection list_tools failed for %s", provider_name
        )

    try:
        result = await provider.retrieve_sources("test", config)
        caps["has_retrieval"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection retrieve_sources failed for %s", provider_name
        )

    try:
        result = await provider.get_audit_trail("test", config)
        caps["has_audit_trail"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection get_audit_trail failed for %s", provider_name
        )

    try:
        result = await provider.get_routing_decision(config)
        caps["has_routing"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection get_routing_decision failed for %s", provider_name
        )

    try:
        result = await provider.get_grounding_report(config)
        caps["has_grounding"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection get_grounding_report failed for %s", provider_name
        )

    try:
        result = await provider.authorize_tool("_test", "_test", config)
        caps["has_authorization"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection authorize_tool failed for %s", provider_name
        )

    try:
        result = await provider.get_governance_architecture(config)
        caps["has_governance_architecture"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection get_governance_architecture failed for %s",
            provider_name,
        )

    try:
        result = await provider.apply_override("_capability_inspection", config)
        caps["has_override_mechanism"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection apply_override failed for %s", provider_name
        )

    try:
        result = await provider.get_configuration_version(config)
        caps["has_configuration_versioning"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection get_configuration_version failed for %s",
            provider_name,
        )

    try:
        result = await provider.get_confidence("_capability_inspection", config)
        caps["has_confidence_scoring"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection get_confidence failed for %s", provider_name
        )

    try:
        result = await provider.route_to_human("_capability_inspection", 0.0, config)
        caps["has_human_routing"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection route_to_human failed for %s", provider_name
        )

    try:
        probe_feed = OutcomeMetricFeed(
            metric_name="_capability_inspection",
            declared_target=0.0,
            realised_series=[0.0],
            higher_is_better=True,
        )
        result = await provider.reconcile_outcome(probe_feed, config)
        caps["has_outcome_reconciliation"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection reconcile_outcome failed for %s", provider_name
        )

    try:
        probe_window = DetectionAuditWindow(
            detector_name="_capability_inspection",
            total_scans=1,
            planted_positive_count=1,
            scanner_true_positives=0,
            false_positives=0,
            declared_tpr_floor=0.0,
            declared_fp_ceiling=1.0,
        )
        result = await provider.evaluate_deployment_gate(probe_window, config)
        caps["has_deployment_gate"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection evaluate_deployment_gate failed for %s",
            provider_name,
        )

    try:
        probe_request = ActionConfirmationRequest(
            action_name="_capability_inspection",
            policy_sensitivity="routine",
            confirmation_gate_present=False,
            auto_execution_blocked=False,
            confirmation_authority="none",
        )
        result = await provider.evaluate_confirmation_gate(probe_request, config)
        caps["has_confirmation_gate"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection evaluate_confirmation_gate failed for %s",
            provider_name,
        )

    try:
        probe_activation = StandingActivationRequest(
            automation_name="_capability_inspection",
            activation_kind="interactive",
            grant_record_present=False,
            grantor_still_authorized=True,
            consent_active=True,
            current_policy_version="0",
            authority_expires_at_activation=0,
            current_activation_index=0,
        )
        result = await provider.revalidate_activation_authority(
            probe_activation, config
        )
        caps["has_activation_revalidation"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection revalidate_activation_authority failed for %s",
            provider_name,
        )

    try:
        probe_record = TrainingRecordCandidate(
            record_name="_capability_inspection",
            record_origin="human_authored",
            manifest_present=False,
            manifest_resolves=False,
            instruction_payload_present=False,
            canary_trigger_present=False,
            duplicate_cluster_size=1,
            duplicate_cluster_threshold=1,
        )
        result = await provider.screen_training_corpus(probe_record, config)
        caps["has_corpus_screening"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection screen_training_corpus failed for %s",
            provider_name,
        )

    try:
        probe_specification = TrainingSpecificationCandidate(
            run_name="_capability_inspection",
            declared_disposition="house_style_adherence",
            objective_text="_capability_inspection",
            review_policy_id="_capability_inspection",
            open_weights_release=False,
        )
        result = await provider.screen_training_specification(
            probe_specification, config
        )
        caps["has_training_specification_screening"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection screen_training_specification failed for %s",
            provider_name,
        )

    try:
        probe_claim = ResponseIdentityClaim(
            response_name="_capability_inspection",
            declared_model_id="_declared",
            declared_model_version="1",
            declared_safety_tier=0,
            served_model_id="_declared",
            served_model_version="1",
            served_safety_tier=0,
            attestation_present=False,
            attestation_resolves=False,
            fingerprint_probe_conclusive=False,
        )
        result = await provider.attest_response_identity(probe_claim, config)
        caps["has_identity_attestation"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection attest_response_identity failed for %s",
            provider_name,
        )

    try:
        probe_handoff = DelegationHandoffClaim(
            delegation_name="_capability_inspection",
            crosses_org_boundary=True,
            parent_scope=("_probe",),
            subtask_required_scope=("_probe",),
            proposed_delegated_scope=("_probe",),
            expected_counterparty_id="_counterparty",
            counterparty_attestation_present=False,
            counterparty_attestation_resolves=False,
        )
        result = await provider.attenuate_delegation_scope(probe_handoff, config)
        caps["has_delegation_attenuation"] = result is not None
    except _CAPABILITY_INSPECTION_EXPECTED_ERRORS:
        _logger.exception(
            "Capability inspection attenuate_delegation_scope failed for %s",
            provider_name,
        )

    caps["has_rate_limit_observability"] = bool(provider.surfaces_rate_limit_errors)

    return ProviderCapabilities(**caps)