Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
import { createRequire } from "node:module";
export * from "./algorithm.js";
export { TranslationEngine, TranslationError, WireFormat } from "./translation.js";
/** Opaque routing state. Store it unchanged and pass it back on the same branch. */
export type CompositeState = { readonly [key: string]: unknown };
export interface CompositeOptions {
targets: { capable: string; efficient: string; classifier: string };
classifier: {
classify_trigger: "user_turn" | "new_session";
fail_open: boolean;
/** Selects capable-first when the `capable` probability is above this. Defaults to 0.6. */
cutoff?: number;
};
/** Stage profiles. The classifier's tier selects one for the turn. */
stage: { efficient_first: StageProfile; capable_first: StageProfile };
}
export interface StageProfile {
/** Confidence the tool signals need to switch tiers. */
threshold: number;
/** Trailing tool results scored. Defaults to 3. */
recent_window?: number;
/** Requests kept on capable after an escalation. Defaults to 2. Set 0 to disable, -1 to pin. */
capable_hold_turns?: number;
/** Defaults to -1 for capable-first, otherwise 0. Set 0 to disable the hold, -1 to pin. */
efficient_hold_turns?: number;
/** System prompt for requests routed to the capable model. */
capable_system_prompt?: string;
}
export interface Selection {
model: string;
/** Present only when the state changed. */
state?: CompositeState;
/** Prepend this to the system prompt of the request sent to `model`. */
systemPrompt?: string;
}
/** Text and tool activity used for routing, in Switchyard's message format. */
export type RoutingContent =
| { type: "text"; text: string }
| { type: "tool_call"; id: string; name: string; arguments: Record<string, unknown> }
| { type: "tool_result"; tool_call_id: string; content: readonly RoutingContent[]; is_error: boolean };
export interface RoutingMessage {
role: "system" | "developer" | "user" | "assistant" | "tool";
content: readonly RoutingContent[];
}
export interface DecisionRequest {
model?: string | null;
input: RoutingMessage[];
questions: Record<string, DecisionQuestion>;
}
export interface DecisionQuestion {
instructions: string;
kind:
| { type: "boolean"; data: { true_description?: unknown; false_description?: unknown } }
| { type: "choice"; data: { options: { id: string; description: string }[] } }
| { type: "score"; data: { levels: unknown[] } };
}
export interface DecisionResponse {
id?: string | null;
model?: string | null;
answers: Record<string, DecisionAnswer>;
usage?: {
input_tokens?: number | null;
cached_input_tokens?: number | null;
cache_creation_input_tokens?: number | null;
output_tokens?: number | null;
total_tokens?: number | null;
reasoning_tokens?: number | null;
};
}
export interface DecisionAnswer {
value:
| { type: "boolean"; data: { type: "value"; data: boolean } | { type: "probability_true"; data: number } }
| { type: "choice"; data: { selected: string; probabilities?: Record<string, number> | null } }
| { type: "score"; data: { value: number; probabilities?: number[] | null } };
provider_confidence?: number | null;
}
type NativeStep =
| { kind: "decision"; request: string }
| { kind: "done"; model: string; state: string | null; systemPrompt: string | null };
interface NativeRun {
next(response: string | null, error: string | null): Promise<NativeStep>;
close(): void;
}
interface NativeComposite {
start(messages: string, state: string | null): NativeRun;
}
const native: { NativeComposite: new (options: string) => NativeComposite } =
createRequire(import.meta.url)("./switchyard.node");
function abortError(): Error {
return Object.assign(new Error("Routing cancelled"), { name: "AbortError", code: "ABORT_ERR" });
}
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function publicError(error: unknown, cause?: unknown): Error {
const match = /^(ERR_CONFIG|ERR_INVALID_REQUEST|ERR_ROUTING|ABORT_ERR): (.*)$/s.exec(errorMessage(error));
if (match?.[1] === "ABORT_ERR") return abortError();
const code = match?.[1] ?? "ERR_ROUTING";
return Object.assign(new Error(match?.[2] ?? errorMessage(error),code === "ERR_ROUTING" ? { cause } : undefined), { code });
}
function json(value: unknown, code = "ERR_INVALID_REQUEST"): string {
try {
const result = JSON.stringify(value);
if (result === undefined) throw new Error("Expected a JSON value");
return result;
} catch (error) {
throw new Error(`${code}: ${errorMessage(error)}`);
}
}
/** Reusable composite router. The caller performs decision calls and stores branch state. */
export class CompositeRouter {
readonly #native: NativeComposite;
private constructor(options: CompositeOptions) {
this.#native = new native.NativeComposite(json(options, "ERR_CONFIG"));
}
static create(options: CompositeOptions): CompositeRouter {
try {
return new CompositeRouter(options);
} catch (error) {
throw publicError(error);
}
}
async route(messages: readonly RoutingMessage[], options: {
state?: CompositeState;
decide(request: DecisionRequest): Promise<DecisionResponse>;
signal?: AbortSignal;
}): Promise<Selection> {
const { signal } = options;
if (signal?.aborted) throw abortError();
let run: NativeRun | undefined;
let decisionError: unknown;
let onAbort!: () => void;
const aborted = new Promise<never>((_, reject) => {
onAbort = () => {
run?.close();
reject(abortError());
};
});
try {
run = this.#native.start(json(messages), options.state === undefined ? null : json(options.state));
signal?.addEventListener("abort", onAbort, { once: true });
let response: string | null = null;
let failure: string | null = null;
for (;;) {
const step = await Promise.race([run.next(response, failure), aborted]);
if (signal?.aborted) throw abortError();
if (step.kind === "done") {
const selection: Selection = { model: step.model };
if (step.state !== null) selection.state = JSON.parse(step.state);
if (step.systemPrompt !== null) selection.systemPrompt = step.systemPrompt;
return selection;
}
response = null;
failure = null;
let answer: DecisionResponse;
try {
answer = await Promise.race([options.decide(JSON.parse(step.request)), aborted]);
} catch (error) {
if (signal?.aborted) throw abortError();
decisionError = error;
failure = errorMessage(error);
continue;
}
if (signal?.aborted) throw abortError();
response = json(answer);
}
} catch (error) {
if (signal?.aborted) throw abortError();
throw publicError(error, decisionError);
} finally {
signal?.removeEventListener("abort", onAbort);
run?.close();
}
}
}