Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
11.8 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Minimal bindings for Rust-owned libsy algorithms."""

from __future__ import annotations

from collections.abc import Mapping
from typing import TYPE_CHECKING, Any

from switchyard_rust._native import load_native

_EXPORTS = frozenset(
    {
        "Algorithm",
        "AlgorithmFactory",
        "ConfiguredAlgorithm",
        "ContextWindowExceededError",
        "CustomClassifierConfig",
        "DecisionCall",
        "DeescalationConfig",
        "EscalationClassifierConfig",
        "LibsyError",
        "LlmClassifierConfig",
        "LlmFallback",
        "LlmResponse",
        "ModelCall",
        "OutcomeMetadata",
        "RoutingOutcome",
        "Step",
        "TaskClassifierConfig",
        "llm_classifier",
        "llm_task_classifier",
        "noop",
        "random",
        "stage_router",
    }
)

if TYPE_CHECKING:
    from collections.abc import AsyncIterator, Awaitable, Sequence
    from typing import ClassVar, Literal, final

    class LibsyError(RuntimeError): ...

    class ContextWindowExceededError(RuntimeError): ...

    class LlmResponse:
        """A normalized aggregate response or live normalized event stream."""

        @final
        class Agg:
            __match_args__: ClassVar[tuple[Literal["response"]]] = ("response",)
            response: dict[str, object]

            def __init__(self, response: Mapping[str, object]) -> None: ...

        @final
        class Stream:
            __match_args__: ClassVar[tuple[Literal["stream"]]] = ("stream",)
            stream: AsyncIterator[dict[str, object]]

            def __init__(self, stream: AsyncIterator[Mapping[str, object]]) -> None: ...

    @final
    class CustomClassifierConfig:
        """Configure schema-validated routing across runtime model groups.

        ``max_output_tokens`` must be positive. Enabling ``message_hash_fallback``
        requires ``session_affinity``.
        """

        def __init__(
            self,
            prompt: str,
            response_schema: Mapping[str, object],
            selector: str,
            *,
            session_affinity: bool = False,
            message_hash_fallback: bool = False,
            recent_turn_window: int | None = None,
            max_output_tokens: int = 4096,
        ) -> None: ...

    @final
    class DeescalationConfig:
        """Configure when an escalated session may return to the efficient tier.

        ``strong_min_calls`` and ``confirmations`` must be positive.
        ``strong_max_calls``, when set, must not be lower than
        ``strong_min_calls``. Values are validated when the classifier is built,
        and invalid values raise ``ValueError``.
        """

        def __init__(
            self,
            *,
            strong_min_calls: int,
            confirmations: int,
            strong_max_calls: int | None = None,
            weak_cooldown_calls: int = 0,
        ) -> None: ...

    @final
    class EscalationClassifierConfig:
        """Configure response-based escalation between two targets.

        Counts and token limits must be positive, and ``window_message_chars``
        must be at least 50.
        """

        def __init__(
            self,
            *,
            confirmations: int = 2,
            recent_turn_window: int = 28,
            window_message_chars: int = 500,
            deescalation: DeescalationConfig | None = None,
            max_output_tokens: int = 4096,
            prompt: str | None = None,
            response_format_type: Literal["json_schema", "json_object"] = "json_schema",
        ) -> None: ...

    @final
    class ModelCall:
        @property
        def algorithm(self) -> str: ...

        @property
        def request(self) -> dict[str, object]: ...

        @property
        def models(self) -> list[str]: ...

        async def respond(self, work: Awaitable[LlmResponse.Agg | LlmResponse.Stream]) -> None: ...

        async def fail(self, error: BaseException) -> None: ...

    @final
    class DecisionCall:
        @property
        def algorithm(self) -> str: ...

        @property
        def request(self) -> dict[str, object]: ...

        @property
        def model(self) -> str: ...

        async def respond(self, work: Awaitable[Mapping[str, object]]) -> None: ...

        async def fail(self, error: BaseException) -> None: ...

    @final
    class OutcomeMetadata:
        """Read-only outcome identity and optional algorithm evidence."""

        @property
        def outcome_id(self) -> str: ...

        @property
        def algorithm(self) -> str: ...

        @property
        def evidence(self) -> Any | None: ...

    @final
    class RoutingOutcome:
        @property
        def metadata(self) -> OutcomeMetadata | None: ...

        @property
        def selected_model_ids(self) -> list[str]: ...

        @property
        def request(self) -> dict[str, object]: ...

        @property
        def fallback_request(self) -> dict[str, object] | None: ...

        @property
        def candidate_requests(self) -> dict[str, dict[str, object]]: ...

        @property
        def response(self) -> LlmResponse.Agg | LlmResponse.Stream | None: ...

    class Step:
        @final
        class CallModel:
            __match_args__: ClassVar[tuple[Literal["call"]]] = ("call",)
            call: ModelCall

        @final
        class CallDecision:
            __match_args__: ClassVar[tuple[Literal["call"]]] = ("call",)
            call: DecisionCall

        @final
        class Done:
            __match_args__: ClassVar[tuple[Literal["outcome"]]] = ("outcome",)
            outcome: RoutingOutcome

    @final
    class TaskClassifierConfig:
        """Configure capability classification between efficient and capable targets.

        Thresholds must remain within ``[0, 1]``, ``max_output_tokens`` must be
        positive, and ``message_hash_fallback`` requires ``session_affinity``.
        """

        @staticmethod
        def decision(
            *,
            cutoff: float | None = None,
            instructions: str | None = None,
            choices: Sequence[Mapping[str, str]] | None = None,
            apply_cutoff_to: str | None = None,
        ) -> TaskClassifierConfig:
            """Route capable when the ``apply_cutoff_to`` choice probability exceeds ``cutoff``.

            Omitted values use the library defaults: cutoff ``0.6`` and the
            ``capable`` and ``efficient`` choices. Each choice has an ``id`` and a
            ``description``.
            """
            ...

        def __init__(
            self,
            base_threshold: float,
            *,
            threshold_step: float = 0.0,
            session_affinity: bool = False,
            message_hash_fallback: bool = False,
            recent_turn_window: int | None = None,
            max_output_tokens: int = 4096,
            prompt: str | None = None,
            response_format_type: Literal["json_schema", "json_object"] = "json_schema",
        ) -> None: ...

    class LlmClassifierConfig:
        """Select one supported LLM classifier mode.

        Target names and each nested mode configuration must satisfy the selected
        classifier's invariants.
        """

        @staticmethod
        def capability(
            *,
            config: TaskClassifierConfig,
        ) -> LlmClassifierConfig:
            """Route by predicted task capability."""
            ...

        @staticmethod
        def escalation(
            *,
            config: EscalationClassifierConfig,
        ) -> LlmClassifierConfig:
            """Call the efficient target first and escalate judged responses."""
            ...

        @staticmethod
        def custom(
            *,
            default_target: str,
            config: CustomClassifierConfig,
        ) -> LlmClassifierConfig:
            """Route among runtime model groups using a schema-selected label.

            ``default_target`` names the group used when the judge fails or its
            verdict cannot be routed. Alongside ``capable`` and ``efficient`` a
            deployment may define its own group names, so one route can choose
            between more than two models.
            """
            ...

    @final
    class LlmFallback:
        def __init__(
            self,
            *,
            config: TaskClassifierConfig,
        ) -> None: ...

    @final
    class Algorithm:
        def run_stream(
            self,
            request: Mapping[str, object],
            models: Mapping[str, Sequence[str]],
            subagent_models: Mapping[str, Sequence[str]] | None = None,
            headers: Mapping[str, str] | None = None,
        ) -> AsyncIterator[Step.CallModel | Step.CallDecision | Step.Done]: ...

    @final
    class AlgorithmFactory:
        """Build deployment algorithms without provider clients or credential access."""

        @staticmethod
        def from_toml(source: str) -> AlgorithmFactory:
            """Parse a full deployment, raising ValueError for invalid configuration."""
            ...

        def build(self) -> dict[str, ConfiguredAlgorithm]:
            """Build independent handles keyed by the public route IDs."""
            ...

        def build_configured(self) -> dict[str, ConfiguredAlgorithm]:
            """Build handles that check target requirements before yielding host calls."""
            ...

    @final
    class ConfiguredAlgorithm:
        """LLM calls and outcomes use target names; the host resolves provider IDs and settings.

        Decision calls use provider IDs.
        """

        def target_models(self) -> dict[str, dict[str, str]]:
            """Map every possible target key to its ``client`` and provider ``id``.

            Requires ``build_configured()``. This includes ineligible targets;
            execute only candidates returned by a call or final outcome.
            """
            ...

        def run_stream(
            self,
            request: Mapping[str, object],
            *,
            headers: Mapping[str, str] | None = None,
            required_labels: Sequence[str] | None = None,
            task_id: str | None = None,
        ) -> AsyncIterator[Step.CallModel | Step.CallDecision | Step.Done]:
            """Yield host calls; target requirements need a ``build_configured()`` handle.

            Omitted labels mean no assessment; ``[]`` supplies no new requirements.
            Reuse the handle for retained task requirements. The host authorizes
            ``task_id``, which overrides a task ID from headers. Libsy's per-run
            State is not carried forward. Answers and fallbacks must use the
            returned candidates.
            """
            ...

    def noop() -> Algorithm: ...

    def random(
        weights: Sequence[float] | None = None,
        seed: int | None = None,
    ) -> Algorithm: ...

    def llm_classifier(config: LlmClassifierConfig) -> Algorithm:
        """Build a classifier, raising ValueError when its configuration is invalid."""
        ...

    def llm_task_classifier(
        *,
        config: TaskClassifierConfig,
    ) -> Algorithm: ...

    def stage_router(
        *,
        picker: str,
        confidence_threshold: float,
        recent_window: int | None = None,
        escalation_note: str | None = None,
        deescalation_note: str | None = None,
        only_on_wrong_signal_escalation: bool = True,
        capable_system_prompt: str | None = None,
        efficient_system_prompt: str | None = None,
        tool_semantics: dict[str, Sequence[str]] | None = None,
        classifier: LlmFallback | None = None,
    ) -> Algorithm: ...


def __getattr__(name: str) -> object:
    if name in _EXPORTS:
        native: Any = load_native()
        return getattr(native.libsy, name)
    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")


__all__ = sorted(_EXPORTS)