Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(runner): retain target requirements by task (#931)
Alex Fournierc365efe83282
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex 1e75496bf..1093ecbfd 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -183,6 +183,7 @@ struct TargetEligibilityConfig {enum RequirementScope {#[default]Request,+ Task,}struct TargetPromptPolicy {@@ -661,11 +662,10 @@ impl DeploymentConfig {let mut route = Route::from_lane(standard, caller_auth, capabilities, route_targets).with_infra(config.infra.clone());if let Some(eligibility) = &config.eligibility {- let policy = match eligibility.scope {- RequirementScope::Request => {- EligibilityPolicy::new(eligibility.require_explicit_requirements)- }- };+ let policy = EligibilityPolicy::new(+ eligibility.require_explicit_requirements,+ matches!(eligibility.scope, RequirementScope::Task),+ );route = route.with_eligibility(policy,EligibleLanes {@@ -2016,8 +2016,8 @@ secondary = ["approved_secondary"]"unknown field `require_explicit_requirement`",),(- format!("{configured}\nscope = 'task'\n"),- "unknown variant `task`",+ format!("{configured}\nscope = 'tenant'\n"),+ "unknown variant `tenant`",),] {let message = error_message(&invalid);diff --git a/crates/switchyard-runner/src/eligibility.rs b/crates/switchyard-runner/src/eligibility.rsindex ee770d478..1a0f3aa90 100644--- a/crates/switchyard-runner/src/eligibility.rs+++ b/crates/switchyard-runner/src/eligibility.rs@@ -1,10 +1,11 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Trusted target requirements for one request.+//! Trusted target requirements and bounded task retention.-use std::collections::BTreeSet;+use std::collections::{BTreeSet, HashMap};+use parking_lot::Mutex;use serde_json::Value;use switchyard_protocol::{LlmClientError, Request, WireFormat};@@ -12,25 +13,52 @@ const REQUIREMENTS_KEY: &str = "switchyard.internal.required_target_labels";const MAX_LABELS: usize = 64;const MAX_LABEL_BYTES: usize = 128;const MAX_REQUIREMENTS_BYTES: usize = MAX_LABELS * (MAX_LABEL_BYTES * 6 + 4) + 2;+const MAX_TASK_ID_BYTES: usize = 512;+const MAX_TASKS: usize = 4_096;const RESPONSES_STATE_FIELDS: [&str; 2] = ["previous_response_id", "conversation"];pub(crate) struct EligibilityPolicy {require_explicit_requirements: bool,+ tasks: Option<Mutex<HashMap<String, BTreeSet<String>>>>,}impl EligibilityPolicy {- pub(crate) fn new(require_explicit_requirements: bool) -> Self {+ pub(crate) fn new(require_explicit_requirements: bool, task_scope: bool) -> Self {Self {require_explicit_requirements,+ tasks: task_scope.then(Mutex::default),}}+ // Retain valid requirements before checking provider-held context for this turn.pub(crate) fn admit(&self, request: &Request) -> Result<TargetRequirements, LlmClientError> {let supplied = target_requirements(request)?;if self.require_explicit_requirements && supplied.is_none() {return Err(invalid("route requires explicit target requirements"));}- let required_labels = supplied.unwrap_or_default();+ let mut required_labels = supplied.unwrap_or_default();+ if let Some(tasks) = &self.tasks {+ let task_id = request+ .metadata+ .as_ref()+ .and_then(|metadata| metadata.task_id.as_deref())+ .filter(|id| !id.is_empty() && id.len() <= MAX_TASK_ID_BYTES && id.trim() == *id)+ .ok_or_else(|| invalid("target eligibility needs a valid task id"))?;+ // Validate and merge under one lock so concurrent turns cannot lose requirements.+ let mut tasks = tasks.lock();+ if let Some(retained) = tasks.get_mut(task_id) {+ if retained.len() + required_labels.difference(retained).count() > MAX_LABELS {+ return Err(invalid("too many target requirement labels"));+ }+ retained.append(&mut required_labels);+ required_labels.clone_from(retained);+ } else if !required_labels.is_empty() {+ if tasks.len() >= MAX_TASKS {+ return Err(invalid("target requirement task capacity exhausted"));+ }+ tasks.insert(task_id.to_owned(), required_labels.clone());+ }+ }validate_request_context(request)?;Ok(TargetRequirements { required_labels })}@@ -109,7 +137,7 @@ fn target_requirements(request: &Request) -> Result<Option<BTreeSet<String>>, LlOk(labels)}-/// Validated target requirements supplied by the caller.+/// Effective target requirements after combining caller requirements and retained task state.pub(crate) struct TargetRequirements {required_labels: BTreeSet<String>,}@@ -163,8 +191,9 @@ mod tests {use super::*;- fn request(labels: &[&str]) -> Request {+ fn task(id: &str, labels: &[&str]) -> Request {let mut request = Request::default();+ request.metadata.get_or_insert_default().task_id = Some(id.into());require_target_labels(&mut request, labels).expect("trusted requirements");request}@@ -190,16 +219,17 @@ mod tests {#[test]fn requirements_are_normalized_and_only_accumulate() {- let policy = EligibilityPolicy::new(false);- let mut request = request(&["customer_data", "private", "private"]);+ let policy = EligibilityPolicy::new(false, false);+ let mut request = task("one", &["customer_data", "private", "private"]);require_target_labels(&mut request, ["approved"]).expect("additional requirement");require_target_labels(&mut request, [] as [&str; 0]).expect("cannot clear requirements");assert_labels(&policy, &request, &["approved", "customer_data", "private"]);+ assert_labels(&policy, &task("one", &[]), &[]);}#[test]fn invalid_requirements_do_not_replace_valid_requirements() {- let mut request = request(&["private"]);+ let mut request = task("one", &["private"]);for invalid_label in [""," ",@@ -216,46 +246,98 @@ mod tests {for encoded in ["null", "{}", "[1]", "[\"\"]", "["] {metadata(&mut request).insert(REQUIREMENTS_KEY.into(), encoded.into());assert!(has_target_requirements(&request));- assert!(EligibilityPolicy::new(false).admit(&request).is_err());+ assert!(+ EligibilityPolicy::new(false, false)+ .admit(&request)+ .is_err()+ );assert!(require_target_labels(&mut request, ["private"]).is_err());}}#[test]fn required_labels_distinguish_missing_from_empty() {- let policy = EligibilityPolicy::new(true);+ let policy = EligibilityPolicy::new(true, false);assert!(policy.admit(&Request::default()).is_err());let decision = policy- .admit(&request(&[]))+ .admit(&task("one", &[])).expect("explicit empty requirements");assert!(decision.required_labels().is_empty());}+ #[test]+ fn task_requirements_accumulate_without_affecting_other_tasks() {+ let policy = EligibilityPolicy::new(false, true);+ policy+ .admit(&task("one", &["private"]))+ .expect("first requirement");+ let decision = policy+ .admit(&task("one", &["customer_data"]))+ .expect("union");+ assert_eq!(+ decision.required_labels(),+ &labels(&["private", "customer_data"])+ );+ assert_labels(&policy, &task("one", &[]), &["private", "customer_data"]);+ assert_labels(&policy, &task("two", &[]), &[]);+ let mut absent = Request::default();+ absent.metadata.get_or_insert_default().task_id = Some("one".into());+ assert_labels(&policy, &absent, &["private", "customer_data"]);++ let explicit = EligibilityPolicy::new(true, true);+ assert_labels(&explicit, &task("one", &["private"]), &["private"]);+ assert!(explicit.admit(&absent).is_err());+ assert_labels(&explicit, &task("one", &[]), &["private"]);+ }++ // Invalid labels and task identities must not alter retained requirements.+ #[test]+ fn task_requirements_validate_signals_and_ids_before_retaining() {+ let policy = EligibilityPolicy::new(false, true);+ assert_labels(&policy, &task("one", &["private"]), &["private"]);+ for encoded in ["null", "{}", "[1]", "[\"\"]", "["] {+ let mut request = task("one", &[]);+ metadata(&mut request).insert(REQUIREMENTS_KEY.into(), encoded.into());+ assert!(policy.admit(&request).is_err());+ assert_labels(&policy, &task("one", &[]), &["private"]);+ }+ assert!(policy.admit(&Request::default()).is_err());+ for id in ["", " ", " one", "one ", &"x".repeat(MAX_TASK_ID_BYTES + 1)] {+ assert!(policy.admit(&task(id, &["customer_data"])).is_err());+ }+ assert_eq!(policy.tasks.as_ref().unwrap().lock().len(), 1);+ assert_labels(+ &policy,+ &task(&"x".repeat(MAX_TASK_ID_BYTES), &["private"]),+ &["private"],+ );+ }+#[test]fn rejected_provider_history_does_not_erase_valid_requirements() {- let policy = EligibilityPolicy::new(false);+ let policy = EligibilityPolicy::new(false, true);for field in RESPONSES_STATE_FIELDS {- let mut request = request(&["private"]);+ let mut request = task(field, &["private"]);request.llm_request.extensions.fields.insert(field.into(), Value::String("hidden".into()));assert!(policy.admit(&request).is_err());- request.llm_request.extensions.fields.remove(field);- assert_labels(&policy, &request, &["private"]);- let mut preserved = self::request(&[]);+ assert_labels(&policy, &task(field, &[]), &["private"]);+ let mut preserved = task(field, &["customer_data"]);preserved.llm_request.preservation.requests.insert(WireFormat::OpenAiResponses.as_str().into(),serde_json::json!({field: "hidden"}),);assert!(policy.admit(&preserved).is_err());+ assert_labels(&policy, &task(field, &[]), &["private", "customer_data"]);}}#[test]- fn label_limits_reject_without_replacing_valid_requirements() {- let mut request = request(&["private"]);+ fn label_and_task_limits_reject_without_partially_retaining() {+ let mut request = task("one", &["private"]);assert!(require_target_labels(&mut request, (0..MAX_LABELS).map(|index| index.to_string())).is_err()@@ -264,5 +346,28 @@ mod tests {target_requirements(&request).unwrap(),Some(labels(&["private"])));+ let policy = EligibilityPolicy::new(false, true);+ let retained = (0..MAX_LABELS).map(|index| index.to_string()).collect();+ policy.tasks.as_ref().unwrap().lock().extend(+ (0..MAX_TASKS).map(|index| (index.to_string(), BTreeSet::from(["private".into()]))),+ );+ assert!(policy.admit(&task("new", &["private"])).is_err());+ assert_labels(&policy, &task("new", &[]), &[]);+ assert_eq!(policy.tasks.as_ref().unwrap().lock().len(), MAX_TASKS);+ policy+ .tasks+ .as_ref()+ .unwrap()+ .lock()+ .insert("0".into(), retained);+ assert!(policy.admit(&task("0", &["customer_data"])).is_err());+ assert_eq!(+ policy+ .admit(&task("0", &[]))+ .unwrap()+ .required_labels()+ .len(),+ MAX_LABELS+ );}}diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rsindex 125508f35..4c8061eab 100644--- a/crates/switchyard-runner/src/route.rs+++ b/crates/switchyard-runner/src/route.rs@@ -488,13 +488,13 @@ mod tests {}#[test]- fn infra_spillover_stays_within_request_requirements() {+ fn infra_rechecks_task_requirements_and_spills_only_to_approved_models() {let algorithm = "type = 'llm_classifier'\nclassifier_target = 'judge'\n\strong_target = 'capable'\nweak_target = 'fast'\nbase_threshold = 0.5";let source = format!("{}\n[routes.test.infra]\nmax_signal_age_ms = 1000\n\min_waiting_requests = 4\nmax_capable_inflight = 2\n",- config(algorithm)+ config(algorithm).replace("scope = 'request'", "scope = 'task'"));let runner = Runner::from_toml(&source).expect("eligible queue policy");let route = runner.route("route").expect("route");@@ -521,8 +521,13 @@ mod tests {}).collect();- let mut stale =- RoutingOutcome::route_to("fast".into(), vec![], request("request", &["private"]));+ let mut stale = RoutingOutcome::route_to("fast".into(), vec![], request("task", &[]));+ route+ .select_lane(&request("task", &["private"]))+ .expect("retain private");+ route+ .select_lane(&request("task", &["customer_data"]))+ .expect("accumulate requirements");let error = route.apply_infra(&mut stale, &signals).expect_err("stale decision");@@ -531,12 +536,11 @@ mod tests {"{error}");- for (labels, initial, expected) in [- (&[] as &[&str], "fast", "capable"),- (&["private"], "approved_fast", "approved_capable"),+ for (task, initial, expected) in [+ ("other", "fast", "capable"),+ ("task", "approved_fast", "approved_capable"),] {- let mut outcome =- RoutingOutcome::route_to(initial.into(), vec![], request("request", labels));+ let mut outcome = RoutingOutcome::route_to(initial.into(), vec![], request(task, &[]));assert!(route.apply_infra(&mut outcome, &signals)diff --git a/crates/switchyard-runner/tests/target_eligibility.rs b/crates/switchyard-runner/tests/target_eligibility.rsindex ab1cdd714..566a4d03e 100644--- a/crates/switchyard-runner/tests/target_eligibility.rs+++ b/crates/switchyard-runner/tests/target_eligibility.rs@@ -1,7 +1,7 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Trusted request requirements constrain routing roles and provider calls.+//! Trusted requirements constrain routing roles, provider calls and retained tasks.use serde_json::{Value, json};use switchyard_llm_client::AuxiliaryOperation;@@ -430,6 +430,59 @@ async fn explicit_requirements_apply_only_to_the_current_request() {assert!(models_called(&server).await.is_empty());}+#[tokio::test]+async fn explicit_empty_differs_from_missing_and_task_requirements_only_accumulate() {+ let server = MockServer::start().await;+ let algorithm = STAGE+ .replace("both_fast", "fast")+ .replace("both_capable", "capable");+ let policy = format!("require_explicit_requirements = true\nscope = 'task'\n{ALTERNATIVES}");+ let source = config(&server.uri(), &algorithm, &policy);+ let runner = Runner::from_toml(&source).expect("task-scoped route");+ let route = runner.route("chat").expect("route");+ assert!(route.decide(request("shared", None)).await.is_err());+ for (task, labels, expected) in [+ ("shared", &[] as &[&str], ["fast", "capable"]),+ ("shared", &["private"], ["private_fast", "private_capable"]),+ ("shared", &["customer_data"], ["both_fast", "both_capable"]),+ ("shared", &[], ["both_fast", "both_capable"]),+ ("other", &[], ["fast", "capable"]),+ ] {+ let outcome = route+ .decide(request(task, Some(labels)))+ .await+ .expect("retained task requirements");+ assert_eq!(outcome.selected_model_ids, expected);+ }+ assert!(route.decide(request("shared", None)).await.is_err());++ let unavailable = source.replace("[\"private\",\"customer_data\"]", "[\"private\"]");+ let runner =+ Runner::from_toml(&unavailable).expect("valid config with unavailable intersection");+ let route = runner.route("chat").expect("route");+ route+ .decide(request("blocked", Some(&["private"])))+ .await+ .expect("initial private task");+ for labels in [&["customer_data"] as &[&str], &["private"], &[]] {+ assert!(+ route+ .decide(request("blocked", Some(labels)))+ .await+ .is_err()+ );+ }+ let other = route+ .decide(request("other", Some(&["private"])))+ .await+ .expect("independent task requirements");+ assert_eq!(+ other.selected_model_ids,+ ["private_fast", "private_capable"]+ );+ assert!(models_called(&server).await.is_empty());+}+#[tokio::test]async fn unapproved_primary_fallback_or_judge_fails_before_provider_io() {let server = MockServer::start().await;diff --git a/docs/internal/target-eligibility.md b/docs/internal/target-eligibility.mdindex 07b13d4a8..fa2b487bc 100644--- a/docs/internal/target-eligibility.md+++ b/docs/internal/target-eligibility.md@@ -51,8 +51,9 @@ capable = ["approved_capable"]supplied requirements, including an empty collection.- `require_explicit_requirements` defaults to `false`. When `true`, every requestneeds a collection; an explicitly empty one counts.-- `scope` is `"request"`, the default and only supported value. Requirements- cover this request and its internal model calls.+- `scope` defaults to `"request"`. Requirements cover this request and its+ internal model calls. `"task"` also retains labels under the host-authorized+ `request.metadata.task_id`.- `target_alternatives` maps configured target names to ordered alternatives;default `{}`. Keys name targets, not router roles. Alternatives must preservetarget kind.@@ -85,16 +86,33 @@ let output = route.execute(request, None).await?;```An empty collection means the host assessed the request and found no requirements.-Omitting the helper supplies no assessment. With the default optional policy,-both allow normal routing. With `require_explicit_requirements = true`, omission-is rejected. Labels accumulate within one request but are not retained across-requests, even when they carry the same task ID.+Omitting the helper supplies no assessment. With request scope and the default+optional policy, both allow normal routing. With+`require_explicit_requirements = true`, omission is rejected. With request scope,+labels accumulate within one request but are not retained across requests, even+when they carry the same task ID.++## Task requirements++Set `scope = "task"` to retain requirements across requests. Every request needs+a trusted `request.metadata.task_id`: a nonempty string of at most 512 bytes,+without surrounding whitespace. The host must authorize task ownership.++Labels only accumulate for the same task ID; different IDs are independent.+For example, a task requiring `private` that later adds `customer_data` requires+both from then on. Missing or empty requirements do not remove retained labels.+If the combined requirements cannot be served, the request fails and the labels+remain retained.++State is in memory and belongs to one route instance. There is no reset, expiry,+replication, restart persistence or revocation. Overlapping in-flight requests+keep the requirements under which they were admitted.## Execution```mermaidflowchart TD- H["Trusted host: request + required labels"] --> A["Runner checks requirements"]+ H["Trusted host: request + required labels"] --> A["Runner checks requirements and adds retained task labels"]A --> P["Resolve eligible roles; filter configured pools"]P -->|"no valid setup"| E["Error: no provider call"]P --> R["Existing routing algorithm"]@@ -107,8 +125,10 @@ targets. Each setup keeps its algorithm, clients, model groups and auxiliarytargets together. Required judges, typed decisions, answer fallbacks and auxiliarycalls must qualify. Provider codecs keep their existing translation role.-A route caches at most 128 additional model setups. Requirements contain at most-64 labels of 128 bytes each. Limits fail closed; routing state is not evicted.+A route retains at most 4,096 restricted tasks and caches at most 128 additional+model setups. Requirements contain at most 64 labels of 128 bytes each, including+the combined labels retained for a task. Limits fail closed; requirements are not+cleared and routing state is not evicted.Provider-held Responses continuation state and forwarded caller authenticationare rejected on eligibility routes.