Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
13.6 KB
Raw Blame
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Complete Transformers prefill and checkpoint inference for the Rust crate."""

from __future__ import annotations

from pathlib import Path
from typing import Any

import numpy as np

EXPECTED_FORMAT_VERSION = 1
EXPECTED_ENCODER_VIEW = "task_prompt_only"
EXPECTED_POOLING = "mean of independently standardized selected layers"
EXPECTED_ACTIVATION = "ReLU"
EXPECTED_ENSEMBLE_REDUCTION = "mean(sigmoid(logits))"


def _expect_equal(name: str, actual: Any, expected: Any) -> None:
    if actual != expected:
        raise ValueError(f"unsupported checkpoint {name}: expected {expected!r}, got {actual!r}")


def _validate_checkpoint_contract(checkpoint: dict[str, Any]) -> None:
    _expect_equal("format_version", checkpoint["format_version"], EXPECTED_FORMAT_VERSION)

    encoder = checkpoint["encoder"]
    architecture = checkpoint["architecture"]
    pipeline = checkpoint["feature_pipeline"]
    _expect_equal("encoder view", encoder["view"], EXPECTED_ENCODER_VIEW)
    _expect_equal("feature pooling", pipeline["pooling"], EXPECTED_POOLING)
    _expect_equal("activation", architecture["activation"], EXPECTED_ACTIVATION)
    _expect_equal(
        "ensemble reduction",
        architecture["ensemble_reduction"],
        EXPECTED_ENSEMBLE_REDUCTION,
    )


def _select_dtype(torch: Any, device: str) -> Any:
    if device == "cpu":
        return torch.float32
    if device == "mps":
        return torch.float16
    if torch.cuda.get_device_capability(device)[0] >= 8:
        return torch.bfloat16
    return torch.float16


def _state_value(state: dict[str, Any], key: str) -> Any:
    try:
        return state[key]
    except KeyError as exc:
        raise ValueError(f"checkpoint model state is missing {key!r}") from exc


def _linear_from_state(torch: Any, state: dict[str, Any], prefix: str) -> Any:
    weight = _state_value(state, f"{prefix}.weight")
    bias = _state_value(state, f"{prefix}.bias")
    if len(weight.shape) != 2:
        raise ValueError(
            f"checkpoint {prefix}.weight has invalid shape: expected 2 dimensions, "
            f"got {tuple(weight.shape)!r}"
        )
    output_features, input_features = weight.shape
    if tuple(bias.shape) != (output_features,):
        raise ValueError(
            f"checkpoint {prefix}.bias has invalid shape: expected {(output_features,)!r}, "
            f"got {tuple(bias.shape)!r}"
        )
    return torch.nn.Linear(input_features, output_features)


def _build_router_member(torch: Any, state: dict[str, Any], output_count: int) -> Any:
    class RouterMember(torch.nn.Module):
        def __init__(self) -> None:
            super().__init__()
            self.adapters = torch.nn.ModuleList(
                [_linear_from_state(torch, state, f"adapters.{index}") for index in range(output_count)]
            )
            self.trunk = torch.nn.Sequential(
                _linear_from_state(torch, state, "trunk.0"),
                torch.nn.ReLU(),
            )
            self.heads = torch.nn.ModuleList(
                [_linear_from_state(torch, state, f"heads.{index}") for index in range(output_count)]
            )

        def forward(self, features: Any) -> Any:
            logits = []
            for adapter, head in zip(self.adapters, self.heads, strict=True):
                adapter_output = torch.nn.functional.relu(adapter(features))
                logits.append(head(self.trunk(adapter_output)))
            return torch.cat(logits, dim=1)

    member = RouterMember()
    try:
        member.load_state_dict(state, strict=True)
    except RuntimeError as exc:
        raise ValueError(f"checkpoint model state does not match router head: {exc}") from exc
    member.eval()
    return member


def _detect_device(torch: Any) -> str:
    if torch.cuda.is_available():
        return "cuda"
    if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
        return "mps"
    return "cpu"


def _resolve_device(torch: Any, override: str | None) -> str:
    device = override.lower().strip() if override is not None else _detect_device(torch)
    if device == "cpu":
        return device
    if device == "mps":
        if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
            return device
        raise ValueError("MPS was requested but is not available")
    if device == "cuda":
        if torch.cuda.is_available():
            return device
        raise ValueError("CUDA was requested but is not available")
    if device.startswith("cuda:") and device.removeprefix("cuda:").isdigit():
        index = int(device.removeprefix("cuda:"))
        if torch.cuda.is_available() and index < torch.cuda.device_count():
            return device
        raise ValueError(f"CUDA device {device} is not available")
    raise ValueError(f"Unsupported device: {device}")


class TransformersForward:
    """Run encoder extraction and learned confidence inference in one pass."""

    def __init__(
        self,
        checkpoint_path: str | Path,
        *,
        device: str | None = None,
        cache_dir: str | None = None,
    ) -> None:
        import torch

        checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
        _validate_checkpoint_contract(checkpoint)

        encoder = checkpoint["encoder"]
        pipeline = checkpoint["feature_pipeline"]
        self._torch = torch
        self._model_path = str(encoder["name"])
        self._expected_layers = int(encoder["n_layers"])
        self._hidden_dim = int(encoder["hidden_dim"])
        self._models = [str(model) for model in checkpoint["models"]]
        self._selected_layers = [int(layer) for layer in pipeline["selected_layers"]]
        self._layer_mean = torch.stack(
            [pipeline["layer_mean"][str(layer)].float() for layer in self._selected_layers]
        ).numpy()
        self._layer_std = torch.stack(
            [pipeline["layer_std"][str(layer)].float() for layer in self._selected_layers]
        ).numpy()
        self._scaler_mean = pipeline["scaler_mean"].numpy()
        self._scaler_scale = pipeline["scaler_scale"].numpy()
        self._pca_mean = pipeline["pca_mean"].numpy()
        self._pca_components = pipeline["pca_components"].numpy()
        self._ensemble = [
            _build_router_member(torch, state, len(self._models))
            for state in checkpoint["model_state_dicts"]
        ]
        self._cache_dir = cache_dir
        self._device_override = device
        self._model = None
        self._tokenizer = None

        self._validate_checkpoint_arrays()

    def _validate_checkpoint_arrays(self) -> None:
        if not self._selected_layers:
            raise ValueError("checkpoint selected layers are invalid: expected non-empty, got []")
        if len(set(self._selected_layers)) != len(self._selected_layers):
            raise ValueError(
                "checkpoint selected layers are invalid: expected unique values, "
                f"got {self._selected_layers!r}"
            )
        if not self._models:
            raise ValueError("checkpoint models are invalid: expected non-empty, got []")
        if not self._ensemble:
            raise ValueError("checkpoint ensemble is invalid: expected non-empty, got []")
        expected_shape = (len(self._selected_layers), self._hidden_dim)
        if self._layer_mean.shape != self._layer_std.shape:
            raise ValueError(
                "checkpoint layer normalization shape is inconsistent: "
                f"expected layer_mean shape to equal layer_std shape, got "
                f"{self._layer_mean.shape!r} and {self._layer_std.shape!r}"
            )
        if self._layer_mean.shape != expected_shape:
            raise ValueError(
                "checkpoint layer normalization shape is inconsistent: "
                f"expected {expected_shape!r}, got {self._layer_mean.shape!r}"
            )
        if not bool(np.all(self._layer_std > 0)):
            raise ValueError("checkpoint layer_std is invalid: expected all positive values")
        if not bool(np.all(self._scaler_scale > 0)):
            raise ValueError("checkpoint scaler_scale is invalid: expected all positive values")

    def metadata(self) -> tuple[str, int]:
        """Return the encoder and ordered output count consumed by Rust."""
        return self._model_path, len(self._models)

    def _ensure_loaded(self) -> None:
        if self._model is not None:
            return

        from transformers import AutoModelForCausalLM, AutoTokenizer

        torch = self._torch
        device = _resolve_device(torch, self._device_override)
        dtype = _select_dtype(torch, device)
        model_kwargs: dict[str, Any] = {}
        if device == "cuda":
            model_kwargs["device_map"] = "auto"
        elif device.startswith("cuda:"):
            model_kwargs["device_map"] = {"": device}
        self._tokenizer = AutoTokenizer.from_pretrained(
            self._model_path,
            cache_dir=self._cache_dir,
        )
        if self._tokenizer.pad_token is None:
            self._tokenizer.pad_token = self._tokenizer.eos_token
        causal_model = AutoModelForCausalLM.from_pretrained(
            self._model_path,
            dtype=dtype,
            cache_dir=self._cache_dir,
            **model_kwargs,
        )
        self._model = causal_model.base_model
        del causal_model
        if device == "mps":
            self._model.to(device)
        self._model.eval()
        model_config = getattr(self._model.config, "text_config", self._model.config)
        if (
            model_config.num_hidden_layers != self._expected_layers
            or model_config.hidden_size != self._hidden_dim
        ):
            expected = (self._expected_layers, self._hidden_dim)
            actual = (model_config.num_hidden_layers, model_config.hidden_size)
            raise ValueError(
                "loaded encoder dimensions do not match checkpoint metadata: "
                f"expected {expected!r}, got {actual!r}"
            )

    def forward(self, prompts: list[str], batch_size: int, max_length: int) -> bytes:
        """Return a row-major F32 probability matrix for ordered prompts."""
        self._ensure_loaded()
        if not prompts or any(not prompt for prompt in prompts):
            raise ValueError("prompts must be non-empty")
        if batch_size <= 0 or max_length <= 0:
            raise ValueError("batch_size and max_length must be positive")

        formatted = [
            self._tokenizer.apply_chat_template(
                [{"role": "user", "content": prompt}],
                tokenize=False,
                add_generation_prompt=True,
            )
            for prompt in prompts
        ]
        predictions: list[np.ndarray] = []
        for start in range(0, len(formatted), batch_size):
            inputs = self._tokenizer(
                formatted[start : start + batch_size],
                return_tensors="pt",
                padding=True,
                truncation=True,
                max_length=max_length,
            )
            input_ids = inputs["input_ids"].to(self._model.device)
            attention_mask = inputs["attention_mask"].to(self._model.device)
            with self._torch.inference_mode():
                outputs = self._model(
                    input_ids=input_ids,
                    attention_mask=attention_mask,
                    output_hidden_states=True,
                    use_cache=False,
                )

            layers = []
            for layer in self._selected_layers:
                hidden = outputs.hidden_states[layer].float()
                # The checkpoint consumes the last real token from each selected layer.
                token_mask = attention_mask.to(hidden.device).bool()
                positions = self._torch.arange(hidden.shape[1], device=hidden.device)
                last_token = (
                    positions.expand_as(token_mask).masked_fill(~token_mask, -1).max(dim=1).values
                )
                pooled = hidden[
                    self._torch.arange(hidden.shape[0], device=hidden.device), last_token
                ]
                layers.append(pooled.cpu().numpy())
            predictions.append(self._predict(layers))

        return np.ascontiguousarray(np.concatenate(predictions, axis=0), dtype=np.float32).tobytes()

    def _predict(self, layers: list[np.ndarray]) -> np.ndarray:
        torch = self._torch
        stacked = np.stack(layers)
        if stacked.ndim != 3 or stacked.shape[0] != len(self._selected_layers):
            expected = (len(self._selected_layers), "batch", self._hidden_dim)
            raise ValueError(
                "encoder returned invalid selected-layer features: "
                f"expected {expected!r}, got {stacked.shape!r}"
            )
        standardized = (stacked - self._layer_mean[:, None, :]) / self._layer_std[:, None, :]
        pooled = standardized.mean(axis=0)
        scaled = (pooled - self._scaler_mean) / self._scaler_scale
        features = torch.from_numpy(
            np.ascontiguousarray(
                (scaled - self._pca_mean) @ self._pca_components.T,
                dtype=np.float32,
            )
        )

        members = []
        with torch.inference_mode():
            for member in self._ensemble:
                members.append(torch.sigmoid(member(features)))
        return torch.stack(members).mean(dim=0).contiguous().numpy()

    def unload(self) -> None:
        self._model = None
        self._tokenizer = None
        if self._torch.cuda.is_available():
            self._torch.cuda.empty_cache()
        if hasattr(self._torch, "mps") and self._torch.backends.mps.is_available():
            self._torch.mps.empty_cache()