Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(runner): retain target requirements by task (#931)

Alex Fournierc365efe83282
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 1e75496bf..1093ecbfd 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -183,6 +183,7 @@ struct TargetEligibilityConfig {
enum RequirementScope {
#[default]
Request,
+ Task,
}
struct TargetPromptPolicy {
@@ -661,11 +662,10 @@ impl DeploymentConfig {
let mut route = Route::from_lane(standard, caller_auth, capabilities, route_targets)
.with_infra(config.infra.clone());
if let Some(eligibility) = &config.eligibility {
- let policy = match eligibility.scope {
- RequirementScope::Request => {
- EligibilityPolicy::new(eligibility.require_explicit_requirements)
- }
- };
+ let policy = EligibilityPolicy::new(
+ eligibility.require_explicit_requirements,
+ matches!(eligibility.scope, RequirementScope::Task),
+ );
route = route.with_eligibility(
policy,
EligibleLanes {
@@ -2016,8 +2016,8 @@ secondary = ["approved_secondary"]
"unknown field `require_explicit_requirement`",
),
(
- format!("{configured}\nscope = 'task'\n"),
- "unknown variant `task`",
+ format!("{configured}\nscope = 'tenant'\n"),
+ "unknown variant `tenant`",
),
] {
let message = error_message(&invalid);
diff --git a/crates/switchyard-runner/src/eligibility.rs b/crates/switchyard-runner/src/eligibility.rs
index ee770d478..1a0f3aa90 100644
--- a/crates/switchyard-runner/src/eligibility.rs
+++ b/crates/switchyard-runner/src/eligibility.rs
@@ -1,10 +1,11 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Trusted target requirements for one request.
+//! Trusted target requirements and bounded task retention.
-use std::collections::BTreeSet;
+use std::collections::{BTreeSet, HashMap};
+use parking_lot::Mutex;
use serde_json::Value;
use switchyard_protocol::{LlmClientError, Request, WireFormat};
@@ -12,25 +13,52 @@ const REQUIREMENTS_KEY: &str = "switchyard.internal.required_target_labels";
const MAX_LABELS: usize = 64;
const MAX_LABEL_BYTES: usize = 128;
const MAX_REQUIREMENTS_BYTES: usize = MAX_LABELS * (MAX_LABEL_BYTES * 6 + 4) + 2;
+const MAX_TASK_ID_BYTES: usize = 512;
+const MAX_TASKS: usize = 4_096;
const RESPONSES_STATE_FIELDS: [&str; 2] = ["previous_response_id", "conversation"];
pub(crate) struct EligibilityPolicy {
require_explicit_requirements: bool,
+ tasks: Option<Mutex<HashMap<String, BTreeSet<String>>>>,
}
impl EligibilityPolicy {
- pub(crate) fn new(require_explicit_requirements: bool) -> Self {
+ pub(crate) fn new(require_explicit_requirements: bool, task_scope: bool) -> Self {
Self {
require_explicit_requirements,
+ tasks: task_scope.then(Mutex::default),
}
}
+ // Retain valid requirements before checking provider-held context for this turn.
pub(crate) fn admit(&self, request: &Request) -> Result<TargetRequirements, LlmClientError> {
let supplied = target_requirements(request)?;
if self.require_explicit_requirements && supplied.is_none() {
return Err(invalid("route requires explicit target requirements"));
}
- let required_labels = supplied.unwrap_or_default();
+ let mut required_labels = supplied.unwrap_or_default();
+ if let Some(tasks) = &self.tasks {
+ let task_id = request
+ .metadata
+ .as_ref()
+ .and_then(|metadata| metadata.task_id.as_deref())
+ .filter(|id| !id.is_empty() && id.len() <= MAX_TASK_ID_BYTES && id.trim() == *id)
+ .ok_or_else(|| invalid("target eligibility needs a valid task id"))?;
+ // Validate and merge under one lock so concurrent turns cannot lose requirements.
+ let mut tasks = tasks.lock();
+ if let Some(retained) = tasks.get_mut(task_id) {
+ if retained.len() + required_labels.difference(retained).count() > MAX_LABELS {
+ return Err(invalid("too many target requirement labels"));
+ }
+ retained.append(&mut required_labels);
+ required_labels.clone_from(retained);
+ } else if !required_labels.is_empty() {
+ if tasks.len() >= MAX_TASKS {
+ return Err(invalid("target requirement task capacity exhausted"));
+ }
+ tasks.insert(task_id.to_owned(), required_labels.clone());
+ }
+ }
validate_request_context(request)?;
Ok(TargetRequirements { required_labels })
}
@@ -109,7 +137,7 @@ fn target_requirements(request: &Request) -> Result<Option<BTreeSet<String>>, Ll
Ok(labels)
}
-/// Validated target requirements supplied by the caller.
+/// Effective target requirements after combining caller requirements and retained task state.
pub(crate) struct TargetRequirements {
required_labels: BTreeSet<String>,
}
@@ -163,8 +191,9 @@ mod tests {
use super::*;
- fn request(labels: &[&str]) -> Request {
+ fn task(id: &str, labels: &[&str]) -> Request {
let mut request = Request::default();
+ request.metadata.get_or_insert_default().task_id = Some(id.into());
require_target_labels(&mut request, labels).expect("trusted requirements");
request
}
@@ -190,16 +219,17 @@ mod tests {
#[test]
fn requirements_are_normalized_and_only_accumulate() {
- let policy = EligibilityPolicy::new(false);
- let mut request = request(&["customer_data", "private", "private"]);
+ let policy = EligibilityPolicy::new(false, false);
+ let mut request = task("one", &["customer_data", "private", "private"]);
require_target_labels(&mut request, ["approved"]).expect("additional requirement");
require_target_labels(&mut request, [] as [&str; 0]).expect("cannot clear requirements");
assert_labels(&policy, &request, &["approved", "customer_data", "private"]);
+ assert_labels(&policy, &task("one", &[]), &[]);
}
#[test]
fn invalid_requirements_do_not_replace_valid_requirements() {
- let mut request = request(&["private"]);
+ let mut request = task("one", &["private"]);
for invalid_label in [
"",
" ",
@@ -216,46 +246,98 @@ mod tests {
for encoded in ["null", "{}", "[1]", "[\"\"]", "["] {
metadata(&mut request).insert(REQUIREMENTS_KEY.into(), encoded.into());
assert!(has_target_requirements(&request));
- assert!(EligibilityPolicy::new(false).admit(&request).is_err());
+ assert!(
+ EligibilityPolicy::new(false, false)
+ .admit(&request)
+ .is_err()
+ );
assert!(require_target_labels(&mut request, ["private"]).is_err());
}
}
#[test]
fn required_labels_distinguish_missing_from_empty() {
- let policy = EligibilityPolicy::new(true);
+ let policy = EligibilityPolicy::new(true, false);
assert!(policy.admit(&Request::default()).is_err());
let decision = policy
- .admit(&request(&[]))
+ .admit(&task("one", &[]))
.expect("explicit empty requirements");
assert!(decision.required_labels().is_empty());
}
+ #[test]
+ fn task_requirements_accumulate_without_affecting_other_tasks() {
+ let policy = EligibilityPolicy::new(false, true);
+ policy
+ .admit(&task("one", &["private"]))
+ .expect("first requirement");
+ let decision = policy
+ .admit(&task("one", &["customer_data"]))
+ .expect("union");
+ assert_eq!(
+ decision.required_labels(),
+ &labels(&["private", "customer_data"])
+ );
+ assert_labels(&policy, &task("one", &[]), &["private", "customer_data"]);
+ assert_labels(&policy, &task("two", &[]), &[]);
+ let mut absent = Request::default();
+ absent.metadata.get_or_insert_default().task_id = Some("one".into());
+ assert_labels(&policy, &absent, &["private", "customer_data"]);
+
+ let explicit = EligibilityPolicy::new(true, true);
+ assert_labels(&explicit, &task("one", &["private"]), &["private"]);
+ assert!(explicit.admit(&absent).is_err());
+ assert_labels(&explicit, &task("one", &[]), &["private"]);
+ }
+
+ // Invalid labels and task identities must not alter retained requirements.
+ #[test]
+ fn task_requirements_validate_signals_and_ids_before_retaining() {
+ let policy = EligibilityPolicy::new(false, true);
+ assert_labels(&policy, &task("one", &["private"]), &["private"]);
+ for encoded in ["null", "{}", "[1]", "[\"\"]", "["] {
+ let mut request = task("one", &[]);
+ metadata(&mut request).insert(REQUIREMENTS_KEY.into(), encoded.into());
+ assert!(policy.admit(&request).is_err());
+ assert_labels(&policy, &task("one", &[]), &["private"]);
+ }
+ assert!(policy.admit(&Request::default()).is_err());
+ for id in ["", " ", " one", "one ", &"x".repeat(MAX_TASK_ID_BYTES + 1)] {
+ assert!(policy.admit(&task(id, &["customer_data"])).is_err());
+ }
+ assert_eq!(policy.tasks.as_ref().unwrap().lock().len(), 1);
+ assert_labels(
+ &policy,
+ &task(&"x".repeat(MAX_TASK_ID_BYTES), &["private"]),
+ &["private"],
+ );
+ }
+
#[test]
fn rejected_provider_history_does_not_erase_valid_requirements() {
- let policy = EligibilityPolicy::new(false);
+ let policy = EligibilityPolicy::new(false, true);
for field in RESPONSES_STATE_FIELDS {
- let mut request = request(&["private"]);
+ let mut request = task(field, &["private"]);
request
.llm_request
.extensions
.fields
.insert(field.into(), Value::String("hidden".into()));
assert!(policy.admit(&request).is_err());
- request.llm_request.extensions.fields.remove(field);
- assert_labels(&policy, &request, &["private"]);
- let mut preserved = self::request(&[]);
+ assert_labels(&policy, &task(field, &[]), &["private"]);
+ let mut preserved = task(field, &["customer_data"]);
preserved.llm_request.preservation.requests.insert(
WireFormat::OpenAiResponses.as_str().into(),
serde_json::json!({field: "hidden"}),
);
assert!(policy.admit(&preserved).is_err());
+ assert_labels(&policy, &task(field, &[]), &["private", "customer_data"]);
}
}
#[test]
- fn label_limits_reject_without_replacing_valid_requirements() {
- let mut request = request(&["private"]);
+ fn label_and_task_limits_reject_without_partially_retaining() {
+ let mut request = task("one", &["private"]);
assert!(
require_target_labels(&mut request, (0..MAX_LABELS).map(|index| index.to_string()))
.is_err()
@@ -264,5 +346,28 @@ mod tests {
target_requirements(&request).unwrap(),
Some(labels(&["private"]))
);
+ let policy = EligibilityPolicy::new(false, true);
+ let retained = (0..MAX_LABELS).map(|index| index.to_string()).collect();
+ policy.tasks.as_ref().unwrap().lock().extend(
+ (0..MAX_TASKS).map(|index| (index.to_string(), BTreeSet::from(["private".into()]))),
+ );
+ assert!(policy.admit(&task("new", &["private"])).is_err());
+ assert_labels(&policy, &task("new", &[]), &[]);
+ assert_eq!(policy.tasks.as_ref().unwrap().lock().len(), MAX_TASKS);
+ policy
+ .tasks
+ .as_ref()
+ .unwrap()
+ .lock()
+ .insert("0".into(), retained);
+ assert!(policy.admit(&task("0", &["customer_data"])).is_err());
+ assert_eq!(
+ policy
+ .admit(&task("0", &[]))
+ .unwrap()
+ .required_labels()
+ .len(),
+ MAX_LABELS
+ );
}
}
diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs
index 125508f35..4c8061eab 100644
--- a/crates/switchyard-runner/src/route.rs
+++ b/crates/switchyard-runner/src/route.rs
@@ -488,13 +488,13 @@ mod tests {
}
#[test]
- fn infra_spillover_stays_within_request_requirements() {
+ fn infra_rechecks_task_requirements_and_spills_only_to_approved_models() {
let algorithm = "type = 'llm_classifier'\nclassifier_target = 'judge'\n\
strong_target = 'capable'\nweak_target = 'fast'\nbase_threshold = 0.5";
let source = format!(
"{}\n[routes.test.infra]\nmax_signal_age_ms = 1000\n\
min_waiting_requests = 4\nmax_capable_inflight = 2\n",
- config(algorithm)
+ config(algorithm).replace("scope = 'request'", "scope = 'task'")
);
let runner = Runner::from_toml(&source).expect("eligible queue policy");
let route = runner.route("route").expect("route");
@@ -521,8 +521,13 @@ mod tests {
})
.collect();
- let mut stale =
- RoutingOutcome::route_to("fast".into(), vec![], request("request", &["private"]));
+ let mut stale = RoutingOutcome::route_to("fast".into(), vec![], request("task", &[]));
+ route
+ .select_lane(&request("task", &["private"]))
+ .expect("retain private");
+ route
+ .select_lane(&request("task", &["customer_data"]))
+ .expect("accumulate requirements");
let error = route
.apply_infra(&mut stale, &signals)
.expect_err("stale decision");
@@ -531,12 +536,11 @@ mod tests {
"{error}"
);
- for (labels, initial, expected) in [
- (&[] as &[&str], "fast", "capable"),
- (&["private"], "approved_fast", "approved_capable"),
+ for (task, initial, expected) in [
+ ("other", "fast", "capable"),
+ ("task", "approved_fast", "approved_capable"),
] {
- let mut outcome =
- RoutingOutcome::route_to(initial.into(), vec![], request("request", labels));
+ let mut outcome = RoutingOutcome::route_to(initial.into(), vec![], request(task, &[]));
assert!(
route
.apply_infra(&mut outcome, &signals)
diff --git a/crates/switchyard-runner/tests/target_eligibility.rs b/crates/switchyard-runner/tests/target_eligibility.rs
index ab1cdd714..566a4d03e 100644
--- a/crates/switchyard-runner/tests/target_eligibility.rs
+++ b/crates/switchyard-runner/tests/target_eligibility.rs
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Trusted request requirements constrain routing roles and provider calls.
+//! Trusted requirements constrain routing roles, provider calls and retained tasks.
use serde_json::{Value, json};
use switchyard_llm_client::AuxiliaryOperation;
@@ -430,6 +430,59 @@ async fn explicit_requirements_apply_only_to_the_current_request() {
assert!(models_called(&server).await.is_empty());
}
+#[tokio::test]
+async fn explicit_empty_differs_from_missing_and_task_requirements_only_accumulate() {
+ let server = MockServer::start().await;
+ let algorithm = STAGE
+ .replace("both_fast", "fast")
+ .replace("both_capable", "capable");
+ let policy = format!("require_explicit_requirements = true\nscope = 'task'\n{ALTERNATIVES}");
+ let source = config(&server.uri(), &algorithm, &policy);
+ let runner = Runner::from_toml(&source).expect("task-scoped route");
+ let route = runner.route("chat").expect("route");
+ assert!(route.decide(request("shared", None)).await.is_err());
+ for (task, labels, expected) in [
+ ("shared", &[] as &[&str], ["fast", "capable"]),
+ ("shared", &["private"], ["private_fast", "private_capable"]),
+ ("shared", &["customer_data"], ["both_fast", "both_capable"]),
+ ("shared", &[], ["both_fast", "both_capable"]),
+ ("other", &[], ["fast", "capable"]),
+ ] {
+ let outcome = route
+ .decide(request(task, Some(labels)))
+ .await
+ .expect("retained task requirements");
+ assert_eq!(outcome.selected_model_ids, expected);
+ }
+ assert!(route.decide(request("shared", None)).await.is_err());
+
+ let unavailable = source.replace("[\"private\",\"customer_data\"]", "[\"private\"]");
+ let runner =
+ Runner::from_toml(&unavailable).expect("valid config with unavailable intersection");
+ let route = runner.route("chat").expect("route");
+ route
+ .decide(request("blocked", Some(&["private"])))
+ .await
+ .expect("initial private task");
+ for labels in [&["customer_data"] as &[&str], &["private"], &[]] {
+ assert!(
+ route
+ .decide(request("blocked", Some(labels)))
+ .await
+ .is_err()
+ );
+ }
+ let other = route
+ .decide(request("other", Some(&["private"])))
+ .await
+ .expect("independent task requirements");
+ assert_eq!(
+ other.selected_model_ids,
+ ["private_fast", "private_capable"]
+ );
+ assert!(models_called(&server).await.is_empty());
+}
+
#[tokio::test]
async fn unapproved_primary_fallback_or_judge_fails_before_provider_io() {
let server = MockServer::start().await;
diff --git a/docs/internal/target-eligibility.md b/docs/internal/target-eligibility.md
index 07b13d4a8..fa2b487bc 100644
--- a/docs/internal/target-eligibility.md
+++ b/docs/internal/target-eligibility.md
@@ -51,8 +51,9 @@ capable = ["approved_capable"]
supplied requirements, including an empty collection.
- `require_explicit_requirements` defaults to `false`. When `true`, every request
needs a collection; an explicitly empty one counts.
-- `scope` is `"request"`, the default and only supported value. Requirements
- cover this request and its internal model calls.
+- `scope` defaults to `"request"`. Requirements cover this request and its
+ internal model calls. `"task"` also retains labels under the host-authorized
+ `request.metadata.task_id`.
- `target_alternatives` maps configured target names to ordered alternatives;
default `{}`. Keys name targets, not router roles. Alternatives must preserve
target kind.
@@ -85,16 +86,33 @@ let output = route.execute(request, None).await?;
```
An empty collection means the host assessed the request and found no requirements.
-Omitting the helper supplies no assessment. With the default optional policy,
-both allow normal routing. With `require_explicit_requirements = true`, omission
-is rejected. Labels accumulate within one request but are not retained across
-requests, even when they carry the same task ID.
+Omitting the helper supplies no assessment. With request scope and the default
+optional policy, both allow normal routing. With
+`require_explicit_requirements = true`, omission is rejected. With request scope,
+labels accumulate within one request but are not retained across requests, even
+when they carry the same task ID.
+
+## Task requirements
+
+Set `scope = "task"` to retain requirements across requests. Every request needs
+a trusted `request.metadata.task_id`: a nonempty string of at most 512 bytes,
+without surrounding whitespace. The host must authorize task ownership.
+
+Labels only accumulate for the same task ID; different IDs are independent.
+For example, a task requiring `private` that later adds `customer_data` requires
+both from then on. Missing or empty requirements do not remove retained labels.
+If the combined requirements cannot be served, the request fails and the labels
+remain retained.
+
+State is in memory and belongs to one route instance. There is no reset, expiry,
+replication, restart persistence or revocation. Overlapping in-flight requests
+keep the requirements under which they were admitted.
## Execution
```mermaid
flowchart TD
- H["Trusted host: request + required labels"] --> A["Runner checks requirements"]
+ H["Trusted host: request + required labels"] --> A["Runner checks requirements and adds retained task labels"]
A --> P["Resolve eligible roles; filter configured pools"]
P -->|"no valid setup"| E["Error: no provider call"]
P --> R["Existing routing algorithm"]
@@ -107,8 +125,10 @@ targets. Each setup keeps its algorithm, clients, model groups and auxiliary
targets together. Required judges, typed decisions, answer fallbacks and auxiliary
calls must qualify. Provider codecs keep their existing translation role.
-A route caches at most 128 additional model setups. Requirements contain at most
-64 labels of 128 bytes each. Limits fail closed; routing state is not evicted.
+A route retains at most 4,096 restricted tasks and caches at most 128 additional
+model setups. Requirements contain at most 64 labels of 128 bytes each, including
+the combined labels retained for a task. Limits fail closed; requirements are not
+cleared and routing state is not evicted.
Provider-held Responses continuation state and forwarded caller authentication
are rejected on eligibility routes.