Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(node): support configured target requirements (#1250)
Alex Fournier6e4864d01ad0
diff --git a/bindings/typescript/README.md b/bindings/typescript/README.mdindex e519da679..e3469a782 100644--- a/bindings/typescript/README.md+++ b/bindings/typescript/README.md@@ -41,8 +41,46 @@ const steps = routes["public/main"].runStream(request, { headers });`source` is deployment TOML and `request` is a normalized Switchyard request.The factory validates configuration without resolving credentials or creatingprovider clients. Reuse configured algorithms to retain algorithm-owned state.-Configurations that require target eligibility checks must use Runner admission;-the factory rejects them.+`build()` rejects configurations that require target eligibility checks. Use+`buildConfigured()` to return `ConfiguredRoute` handles with those checks while+keeping model calls in the host:++```ts+const routes = AlgorithmFactory.fromToml(source).buildConfigured();+const coding = routes["switchyard/coding"];+const targetModels = coding.targetModels();+const steps = coding.runStream(request, {+ requiredLabels: ["private", "customer_data"],+ taskId: trustedTaskId,+});+```++The application owns the labels and task identity. Omitted `requiredLabels` means+no assessment was supplied; `[]` explicitly supplies no new requirements.+`runStream()` throws immediately for malformed options or invalid label syntax.+Target eligibility is checked on the first iterator read, before any model call is+yielded; failures reject that read. Handle errors from both creating and reading+the iterator. Routes configured with `scope = "task"`+retain requirements on the reused handle; later requests can add labels but+cannot clear them. `taskId` takes precedence over a task ID derived from headers.+Retained requirements are local to that handle and process.+Handles from `build()` accept only headers; supplied labels (including `[]`) or+`taskId` are rejected.++`coding.targetModels()` maps every possible call target to `{ client, id }`,+including judges, subagents and ordered alternatives. Keys match emitted calls:+logical target names for LLM calls, provider model IDs for Decision calls.+This is a lookup, not an eligible pool; serve only emitted calls and outcome+candidates. `targetModels()` is available only on `ConfiguredRoute` handles from+`buildConfigured()`.++For an emitted model `call`, use the host's existing client registry:++```ts+const { client, id } = targetModels[call.models[0]];+const work = clients[client].call({ ...call.request, model: id });+await call.respond(work);+```The iterator yields Python-compatible routing steps:diff --git a/bindings/typescript/package.json b/bindings/typescript/package.jsonindex f631fea7d..9b6d69a9e 100644--- a/bindings/typescript/package.json+++ b/bindings/typescript/package.json@@ -16,7 +16,7 @@"files": ["dist", "LICENSE", "NOTICE", "README.md"],"scripts": {"build": "node scripts/build.mjs && tsc",- "typecheck": "tsc --noEmit",+ "typecheck": "tsc --noEmit -p tsconfig.test.json","test": "node --test test/*.test.mjs"},"devDependencies": {diff --git a/bindings/typescript/src/algorithm.ts b/bindings/typescript/src/algorithm.tsindex 2c2ba45ec..66169f2a2 100644--- a/bindings/typescript/src/algorithm.ts+++ b/bindings/typescript/src/algorithm.ts@@ -56,6 +56,11 @@ export interface RoutingOutcome {readonly metadata: OutcomeMetadata | null;}+export interface TargetModel {+ readonly client: string;+ readonly id: string;+}+export namespace Step {export class CallModel {constructor(readonly call: ModelCall) {}@@ -96,11 +101,13 @@ interface NativeStep {}interface NativeConfigured {- start(request: string, headers: string | null): NativeReader<NativeStep>;+ start(request: string, headers: string | null, options: string | null): NativeReader<NativeStep>;+ targetModels(): Record<string, TargetModel>;}interface NativeFactory {build(): Record<string, NativeConfigured>;+ buildConfigured(): Record<string, NativeConfigured>;}const native: { NativeAlgorithmFactory: new (source: string) => NativeFactory } =@@ -242,6 +249,13 @@ function decodeStep(step: NativeStep): AlgorithmStep {}let configured: (native: NativeConfigured) => ConfiguredAlgorithm;+let configuredRoute: (native: NativeConfigured) => ConfiguredRoute;++interface RouteRunOptions {+ headers?: Record<string, string>;+ requiredLabels?: readonly string[];+ taskId?: string;+}export class AlgorithmFactory {readonly #native: NativeFactory;@@ -265,12 +279,21 @@ export class AlgorithmFactory {throw publicError(error);}}++ /** Build routes with target requirement checks and target lookup. */+ buildConfigured(): Record<string, ConfiguredRoute> {+ try {+ return Object.fromEntries(Object.entries(this.#native.buildConfigured()).map(([id, route]) => [id, configuredRoute(route)]));+ } catch (error) {+ throw publicError(error);+ }+ }}export class ConfiguredAlgorithm {readonly #native: NativeConfigured;- private constructor(native: NativeConfigured) {+ protected constructor(native: NativeConfigured) {this.#native = native;}@@ -278,11 +301,58 @@ export class ConfiguredAlgorithm {configured = (native) => new ConfiguredAlgorithm(native);}- runStream(request: JsonObject, options: { headers?: Record<string, string> } = {}): AsyncIterableIterator<AlgorithmStep> {+ protected readTargetModels(): Readonly<Record<string, TargetModel>> {+ try {+ return this.#native.targetModels();+ } catch (error) {+ throw publicError(error);+ }+ }++ runStream(request: JsonObject, options: {+ headers?: Record<string, string>;+ } = {}): AsyncIterableIterator<AlgorithmStep> {+ return this.start(request, options);+ }++ protected start(request: JsonObject, options: RouteRunOptions): AsyncIterableIterator<AlgorithmStep> {try {- return reader(this.#native.start(json(request), options.headers === undefined ? null : json(options.headers)), decodeStep);+ if (options.requiredLabels !== undefined && !Array.isArray(options.requiredLabels)) {+ throw new Error("ERR_INVALID_REQUEST: requiredLabels must be an array of strings");+ }+ if (options.taskId !== undefined && typeof options.taskId !== "string") {+ throw new Error("ERR_INVALID_REQUEST: taskId must be a string");+ }+ const requirements = options.requiredLabels === undefined && options.taskId === undefined+ ? null : json({ required_labels: options.requiredLabels, task_id: options.taskId });+ return reader(this.#native.start(+ json(request), options.headers === undefined ? null : json(options.headers), requirements,+ ), decodeStep);} catch (error) {throw publicError(error);}}}++/** A configured route that checks target requirements before routing. */+export class ConfiguredRoute extends ConfiguredAlgorithm {+ private constructor(native: NativeConfigured) {+ super(native);+ }++ static {+ configuredRoute = (native) => new ConfiguredRoute(native);+ }++ /** Resolve call targets to configured clients and provider IDs, not an eligible pool. */+ targetModels(): Readonly<Record<string, TargetModel>> {+ return this.readTargetModels();+ }++ /** Run with host-supplied requirements and a trusted task ID. */+ override runStream(+ request: JsonObject, options: RouteRunOptions = {},+ ): AsyncIterableIterator<AlgorithmStep> {+ return this.start(request, options);+ }+}diff --git a/bindings/typescript/test/algorithm.test.mjs b/bindings/typescript/test/algorithm.test.mjsindex b8d4dbf37..d06f9f354 100644--- a/bindings/typescript/test/algorithm.test.mjs+++ b/bindings/typescript/test/algorithm.test.mjs@@ -4,7 +4,7 @@import assert from "node:assert/strict";import { createServer } from "node:http";import { test } from "node:test";-import { AlgorithmFactory, ConfiguredAlgorithm, LibsyError, LlmResponse, Step } from "../dist/index.js";+import { AlgorithmFactory, ConfiguredAlgorithm, ConfiguredRoute, LibsyError, LlmResponse, Step } from "../dist/index.js";const deployment = `schema_version = 1@@ -52,6 +52,40 @@ const answer = { answers: { route: { value: {type: "choice", data: { selected: "capable", probabilities: { capable: 0.9, efficient: 0.1 } },} } } };+function labelledDeployment(scope) {+ const targets = [+ ["private_fast", ["private"]], ["private_capable", ["private"]],+ ["both_fast", ["private", "customer_data"]], ["both_capable", ["private", "customer_data"]],+ ["llm_judge", []], ["private_judge", ["private", "customer_data"]],+ ].map(([name, labels]) => `[targets.${name}]\nid = "upstream/${name}"\nllm_client = "answers"\nlabels = ${JSON.stringify(labels)}\n`).join("");+ const eligibility = [+ ["parent", ""], ["classifier", 'judge = ["private_decision"]'],+ ["llm", 'llm_judge = ["private_judge"]'],+ ].map(([name, judge]) => `+[routes.${name}.eligibility]+require_explicit_requirements = true+scope = "${scope}"+[routes.${name}.eligibility.target_alternatives]+fast = ["private_fast", "both_fast"]+capable = ["private_capable", "both_capable"]+${judge}+`).join("");+ return `${deployment}${targets}+[decision_targets.private_decision]+id = "upstream/private_decision"+decision_client = "judge"+labels = ["private", "customer_data"]+[routes.llm]+id = "public/llm"+type = "llm_classifier"+classifier_target = "llm_judge"+strong_target = "capable"+weak_target = "fast"+fail_open = false+base_threshold = 0.5+${eligibility}`;+}+async function collect(source) {const result = [];for await (const item of source) result.push(item);@@ -92,6 +126,89 @@ test("factory rejects configuration requiring Runner admission", () => {});});+test("configured routes check labels before yielding and retain task requirements", async () => {+ assert.equal(process.env.SWITCHYARD_FACTORY_TEST_KEY_THAT_IS_NOT_SET, undefined);+ const configured = AlgorithmFactory.fromToml(labelledDeployment("task")).buildConfigured();+ const route = configured["public/main"];+ assert.ok(route instanceof ConfiguredAlgorithm);+ assert.ok(route instanceof ConfiguredRoute);+ assert.deepEqual(route.targetModels(), Object.fromEntries([+ "fast", "capable", "private_fast", "private_capable", "both_fast", "both_capable",+ ].map((name) => [name, { client: "answers", id: `upstream/${name}` }])));+ for (const options of [+ { taskId: "missing-assessment" },+ { requiredLabels: [] },+ { requiredLabels: ["unapproved"], taskId: "unapproved" },+ ]) {+ const run = route.runStream(request, options);+ await assert.rejects(run.next(), { code: "ERR_INVALID_REQUEST" });+ assert.deepEqual(await run.next(), { done: true, value: undefined });+ }+ for (const options of [+ { requiredLabels: "private" }, { requiredLabels: null }, { requiredLabels: [1] },+ { requiredLabels: [""] }, { taskId: 1 }, { taskId: null },+ ]) {+ assert.throws(() => route.runStream(request, options), { code: "ERR_INVALID_REQUEST" });+ }+ for (const [taskId, requiredLabels, expected] of [+ ["shared", ["private"], "private_fast"],+ ["shared", ["customer_data"], "both_fast"],+ ["shared", [], "both_fast"],+ ["other", [], "fast"],+ ]) {+ const steps = await collect(route.runStream(request, {+ requiredLabels, taskId, headers: { "x-switchyard-task-id": "shared" },+ }));+ assert.equal(steps[0].outcome.selected_model_ids[0], expected);+ }+ for (const options of [+ { requiredLabels: [] },+ { taskId: "trusted", headers: { "x-switchyard-task-id": "from-header" } },+ ]) {+ assert.throws(() => routes["public/main"].runStream(request, options), {+ code: "ERR_INVALID_REQUEST", message: /buildConfigured/,+ });+ }+ assert.equal("targetModels" in routes["public/main"], false);+ await assert.rejects(configured["public/advisor"].runStream(request, { requiredLabels: [] }).next(), {+ code: "ERR_INVALID_REQUEST",+ });+});++test("configured routes yield approved LLM and Decision judges for the host", async (t) => {+ const configured = AlgorithmFactory.fromToml(labelledDeployment("request")).buildConfigured();+ for (const decision of [false, true]) {+ const route = configured[decision ? "public/classify" : "public/llm"];+ const targets = route.targetModels();+ const run = route.runStream(request, {+ requiredLabels: ["private", "customer_data"],+ });+ t.after(() => run.return());+ const step = (await run.next()).value;+ if (decision) {+ assert.ok(step instanceof Step.CallDecision);+ assert.equal(step.call.model, "upstream/private_decision");+ assert.equal(step.call.request.model, step.call.model);+ assert.deepEqual(targets[step.call.model], { client: "judge", id: "upstream/private_decision" });+ await step.call.respond(Promise.resolve(answer));+ } else {+ assert.ok(step instanceof Step.CallModel);+ assert.deepEqual(step.call.models, ["private_judge"]);+ assert.equal(step.call.request.model, "private_judge");+ const target = targets[step.call.models[0]];+ assert.deepEqual(target, { client: "answers", id: "upstream/private_judge" });+ const response = aggregate(JSON.stringify({+ crux: "bounded task", primary_rule: "SUP-1", capability_boundary: "supported", p_solve: 0.9,+ }));+ response.model = target.id;+ await step.call.respond(Promise.resolve(new LlmResponse.Agg(response)));+ }+ const done = (await run.next()).value;+ assert.deepEqual([...done.outcome.selected_model_ids].sort(), ["both_capable", "both_fast"]);+ assert.deepEqual(await run.next(), { done: true, value: undefined });+ }+});+test("decision calls accept async replies and failures", async (t) => {for (const fail of [false, true]) {const run = routes["public/classify"].runStream(request);diff --git a/bindings/typescript/test/algorithm.types.ts b/bindings/typescript/test/algorithm.types.tsnew file mode 100644index 000000000..2df3b8f37--- /dev/null+++ b/bindings/typescript/test/algorithm.types.ts@@ -0,0 +1,25 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++import { AlgorithmFactory, ConfiguredAlgorithm, ConfiguredRoute } from "../src/index.js";++declare const factory: AlgorithmFactory;++const bare = factory.build()["switchyard/coding"];+bare.runStream({}, { headers: { "x-switchyard-task-id": "task" } });+// @ts-expect-error Bare algorithms have no configured target inventory.+bare.targetModels();+// @ts-expect-error Target requirements need a configured route.+bare.runStream({}, { requiredLabels: [] });+// @ts-expect-error Trusted task IDs need a configured route.+bare.runStream({}, { taskId: "task" });++const route: ConfiguredRoute = factory.buildConfigured()["switchyard/coding"];+route.targetModels()["approved"].id;+route.runStream({}, {+ headers: { "x-switchyard-task-id": "header-task" },+ requiredLabels: ["private", "customer_data"],+ taskId: "trusted-task",+});+const compatible: ConfiguredAlgorithm = route;+compatible.runStream({});diff --git a/bindings/typescript/tsconfig.test.json b/bindings/typescript/tsconfig.test.jsonnew file mode 100644index 000000000..0dd63a9b3--- /dev/null+++ b/bindings/typescript/tsconfig.test.json@@ -0,0 +1,5 @@+{+ "extends": "./tsconfig.json",+ "compilerOptions": { "rootDir": "." },+ "include": ["src/**/*.ts", "test/**/*.types.ts"]+}diff --git a/crates/switchyard-node/src/algorithm.rs b/crates/switchyard-node/src/algorithm.rsindex 432ea6b3d..9bd27b162 100644--- a/crates/switchyard-node/src/algorithm.rs+++ b/crates/switchyard-node/src/algorithm.rs@@ -9,7 +9,7 @@ use std::sync::Arc;use std::sync::atomic::{AtomicBool, Ordering};use futures::future::{AbortHandle, Abortable};-use futures::{Stream, StreamExt};+use futures::{Stream, StreamExt, TryStreamExt};use http::{HeaderMap, HeaderName, HeaderValue};use napi::bindgen_prelude::Promise;use napi_derive::napi;@@ -23,7 +23,9 @@ use switchyard_protocol::{AggLlmResponse, DecisionRequest, DecisionResponse, LlmClientError, LlmRequest, LlmResponse,LlmResponseStream, LlmResponseStreamEvent, Metadata, ModelId, Request, Response,};-use switchyard_runner::AlgorithmFactory;+use switchyard_runner::{+ AlgorithmFactory, ConfiguredAlgorithm, RunnerError, require_target_labels,+};use tokio::sync::mpsc;use crate::{error, parse, routing_error};@@ -53,8 +55,28 @@ impl NativeAlgorithmFactory {(id,NativeConfiguredAlgorithm {- algorithm,- models: Arc::new(models),+ inner: AlgorithmHandle::Bare {+ algorithm,+ models: Arc::new(models),+ },+ },+ )+ })+ .collect())+ }++ #[napi]+ pub fn build_configured(&self) -> napi::Result<BTreeMap<String, NativeConfiguredAlgorithm>> {+ Ok(self+ .inner+ .build_configured()+ .map_err(|err| error("ERR_CONFIG", err))?+ .into_iter()+ .map(|(id, algorithm)| {+ (+ id,+ NativeConfiguredAlgorithm {+ inner: AlgorithmHandle::Governed(Arc::new(algorithm)),},)})@@ -62,20 +84,87 @@ impl NativeAlgorithmFactory {}}+#[derive(Clone)]+enum AlgorithmHandle {+ Bare {+ algorithm: Arc<dyn Algorithm>,+ models: Arc<RuntimeModels>,+ },+ Governed(Arc<ConfiguredAlgorithm>),+}++#[derive(Default, Deserialize)]+#[serde(deny_unknown_fields)]+struct RunOptions {+ required_labels: Option<Vec<String>>,+ task_id: Option<String>,+}++fn admission_error(err: RunnerError) -> napi::Error {+ let code = match &err {+ RunnerError::Configuration { .. } => "ERR_CONFIG",+ RunnerError::Client(LlmClientError::InvalidRequest { .. }) => "ERR_INVALID_REQUEST",+ _ => "ERR_ROUTING",+ };+ error(code, err)+}+#[napi]pub struct NativeConfiguredAlgorithm {- algorithm: Arc<dyn Algorithm>,- models: Arc<RuntimeModels>,+ inner: AlgorithmHandle,+}++#[napi(object)]+pub struct NativeTargetModel {+ pub client: String,+ pub id: String,}#[napi]impl NativeConfiguredAlgorithm {+ #[napi]+ pub fn target_models(&self) -> napi::Result<BTreeMap<String, NativeTargetModel>> {+ let AlgorithmHandle::Governed(algorithm) = &self.inner else {+ return Err(error(+ "ERR_INVALID_REQUEST",+ "target discovery requires buildConfigured()",+ ));+ };+ Ok(algorithm+ .target_models()+ .iter()+ .map(|(name, target)| {+ (+ name.to_string(),+ NativeTargetModel {+ client: target.client.clone(),+ id: target.id.to_string(),+ },+ )+ })+ .collect())+ }+#[napi]pub fn start(&self,request_json: String,headers_json: Option<String>,+ options_json: Option<String>,) -> napi::Result<NativeAlgorithmRun> {+ let options: RunOptions = options_json+ .as_deref()+ .map(parse)+ .transpose()?+ .unwrap_or_default();+ if (options.required_labels.is_some() || options.task_id.is_some())+ && matches!(&self.inner, AlgorithmHandle::Bare { .. })+ {+ return Err(error(+ "ERR_INVALID_REQUEST",+ "target requirements or task IDs require buildConfigured()",+ ));+ }let headers: Option<BTreeMap<String, String>> =headers_json.as_deref().map(parse).transpose()?;let headers = headers@@ -95,19 +184,32 @@ impl NativeConfiguredAlgorithm {.transpose()?;let mut llm_request = parse(&request_json)?;switchyard_translation::normalize_compaction(&mut llm_request, headers.as_ref());- let request = Request {+ let mut request = Request {llm_request,raw_request: None,metadata: headers.map(|headers| Metadata::from_headers(&headers)),};- let algorithm = Arc::clone(&self.algorithm);- let models = Arc::clone(&self.models);+ if let Some(task_id) = options.task_id {+ request.metadata.get_or_insert_default().task_id = Some(task_id);+ }+ if let Some(labels) = options.required_labels {+ require_target_labels(&mut request, labels)+ .map_err(|err| error("ERR_INVALID_REQUEST", err))?;+ }+ let handle = self.inner.clone();// Algorithm startup must run on napi's Tokio runtime.let stream = futures::stream::once(async move {- algorithm.run_stream(request, models, State::default())+ let stream = match handle {+ AlgorithmHandle::Bare { algorithm, models } => {+ Ok(algorithm.run_stream(request, models, State::default()))+ }+ AlgorithmHandle::Governed(algorithm) => algorithm+ .run_stream(request, State::default())+ .map_err(admission_error),+ }?;+ Ok::<_, napi::Error>(stream.map(|step| step.map_err(routing_error)))})- .flatten()- .map(|step| step.map_err(routing_error));+ .try_flatten();Ok(NativeAlgorithmRun {reader: Reader::new(stream),})