Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(routing): add queue-aware task routing to Dynamo PreProc (#1073)
Ayush Agarwal59351b04128d
diff --git a/Cargo.lock b/Cargo.lockindex a57642af3..47965ca9f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5509,6 +5509,8 @@ dependencies = ["envoy-types","futures-util","http",+ "reqwest",+ "serde","serde_json","switchyard-protocol","switchyard-runner",@@ -5516,6 +5518,7 @@ dependencies = ["thiserror 2.0.18","tokio","tokio-stream",+ "toml 1.1.3+spec-1.1.0","tonic","tracing","tracing-subscriber",diff --git a/crates/libsy/README.md b/crates/libsy/README.mdindex 0be56eb61..f163b0e5d 100644--- a/crates/libsy/README.md+++ b/crates/libsy/README.md@@ -41,6 +41,9 @@ drives the stream and performs the terminal answer call, retries, and fallback oThe provider-neutral [`Request`], [`Response`], [`Usage`], and [`LlmResponse`]contracts come from `switchyard-protocol`.+The [Dynamo PreProc example](../../examples/dynamo-preproc/README.md#task-classification-and-queue-based-routing)+adds an optional queue-depth check after task classification.+## Concurrent calls and cancellationAn algorithm can poll several `Driver::call_model` or `Driver::call_decision` futuresdiff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rsindex 00fcbea92..4a07e0cfd 100644--- a/crates/libsy/src/algorithms.rs+++ b/crates/libsy/src/algorithms.rs@@ -10,6 +10,7 @@ pub mod advisor_gate;pub mod composite;mod escalation;pub mod fall_through;+pub mod infra;pub mod llm_class;pub mod noop;pub mod passthrough;diff --git a/crates/libsy/src/algorithms/infra.rs b/crates/libsy/src/algorithms/infra.rsnew file mode 100644index 000000000..87f2e970e--- /dev/null+++ b/crates/libsy/src/algorithms/infra.rs@@ -0,0 +1,197 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Optional queue-based moves from the efficient model to the capable model.++use crate::{LibsyError, Result, RoutingOutcome, RuntimeModels};+use serde::Deserialize;+use std::time::{Duration, Instant};+use switchyard_protocol::{Category, InfraSignals, ModelId};++/// Queue and capacity limits for moving requests to the capable model.+#[derive(Clone, Debug, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct InfraPolicy {+ /// Maximum age of the pool observation.+ pub max_signal_age_ms: u64,+ /// Every efficient worker must have at least this many waiting requests.+ pub min_waiting_requests: u64,+ /// Stop moving requests when observed running or locally tracked capable work reaches this limit.+ pub max_capable_inflight: u64,+}++impl InfraPolicy {+ /// Validate the policy once at configuration load.+ pub fn validate(&self) -> Result<()> {+ if self.max_signal_age_ms == 0+ || self.min_waiting_requests == 0+ || self.max_capable_inflight == 0+ {+ return Err(LibsyError::AlgorithmError {+ message: "infra requires positive limits".into(),+ });+ }+ Ok(())+ }++ fn evaluate(+ &self,+ signals: &InfraSignals,+ source: &ModelId,+ destination: &ModelId,+ ) -> Option<(bool, &'static str)> {+ let source = signals.get(source)?.load.as_ref()?;+ let destination = signals.get(destination)?;+ let load = destination.load.as_ref()?;+ let now = Instant::now();+ if [source, load].iter().any(|pool| {+ now.checked_duration_since(pool.sampled_at)+ .is_none_or(|age| age > Duration::from_millis(self.max_signal_age_ms))+ }) {+ return None;+ }+ if source.min_waiting < self.min_waiting_requests {+ return Some((false, "small_pool_not_queued"));+ }+ if load.max_waiting > 0+ || load.running.max(destination.local_inflight) >= self.max_capable_inflight+ {+ return Some((false, "protect_capable_capacity"));+ }+ Some((true, "small_pool_queued"))+ }++ /// Adjust a decision-only task result. Returns true only when the model changes.+ ///+ /// Hosts must serialize this check with their local in-flight increment, then decrement+ /// when the response ends or is cancelled. Worker selection remains in the serving layer.+ pub fn apply(+ &self,+ outcome: &mut RoutingOutcome,+ models: &RuntimeModels,+ signals: &InfraSignals,+ ) -> Result<bool> {+ if outcome.response.is_some()+ || outcome+ .request+ .metadata+ .as_ref()+ .is_some_and(|m| m.is_subagent_work())+ {+ return Ok(false);+ }+ let ([efficient], [capable]) = (+ models.models_for(&Category::Efficient),+ models.models_for(&Category::Capable),+ ) else {+ return Err(LibsyError::AlgorithmError {+ message: "infra requires one efficient and one capable model".into(),+ });+ };+ if outcome.selected_model_id()? != efficient || efficient == capable {+ return Ok(false);+ }+ let (is_spillover, reason) = self+ .evaluate(signals, efficient, capable)+ .unwrap_or((false, "unknown_or_stale_signals"));+ tracing::info!(task_model = %efficient, alternate_model = %capable, spillover = is_spillover, reason,+ "queue routing decision");+ if is_spillover {+ outcome.selected_model_ids.retain(|id| id != capable);+ outcome.selected_model_ids.insert(0, capable.clone());+ outcome.request.llm_request.model = Some(capable.to_string());+ }+ Ok(is_spillover)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use switchyard_protocol::{ModelInfraSignals, PoolLoad, Request, text_request};++ #[test]+ fn queue_policy_preserves_task_choices() {+ let policy = InfraPolicy {+ max_signal_age_ms: 1000,+ min_waiting_requests: 4,+ max_capable_inflight: 2,+ };+ policy.validate().unwrap();+ let models = RuntimeModels::new(+ [+ (Category::Efficient, vec!["small".into()]),+ (Category::Capable, vec!["large".into()]),+ ]+ .into(),+ );+ let request = Request {+ llm_request: text_request(None, "hello"),+ ..Default::default()+ };+ for case in [+ "busy",+ "one_quiet",+ "capable_queued",+ "capable_running",+ "local_limit",+ "missing",+ "stale",+ "future",+ "below_threshold",+ "at_threshold",+ "capable_task",+ ] {+ let model = |waiting| ModelInfraSignals {+ load: Some(PoolLoad {+ sampled_at: Instant::now(),+ running: 0,+ min_waiting: waiting,+ max_waiting: waiting,+ }),+ ..Default::default()+ };+ let mut source = model(8);+ let mut destination = model(0);+ match case {+ "one_quiet" => source.load.as_mut().unwrap().min_waiting = 0,+ "capable_queued" => destination.load.as_mut().unwrap().max_waiting = 1,+ "capable_running" => destination.load.as_mut().unwrap().running = 2,+ "local_limit" => destination.local_inflight = 2,+ "missing" => source.load = None,+ "stale" => source.load.as_mut().unwrap().sampled_at -= Duration::from_secs(2),+ "future" => destination.load.as_mut().unwrap().sampled_at += Duration::from_secs(2),+ "below_threshold" => source.load.as_mut().unwrap().min_waiting = 3,+ "at_threshold" => source.load.as_mut().unwrap().min_waiting = 4,+ _ => {}+ }+ let signals = [+ (ModelId::from("small"), source),+ (ModelId::from("large"), destination),+ ]+ .into();+ let initial = if case == "capable_task" {+ "large"+ } else {+ "small"+ };+ let mut outcome = RoutingOutcome::route_to(initial.into(), vec![], request.clone());+ let is_spillover = matches!(case, "busy" | "at_threshold");+ assert_eq!(+ policy.apply(&mut outcome, &models, &signals).unwrap(),+ is_spillover,+ "{case}"+ );+ assert_eq!(+ outcome.selected_model_id().unwrap().as_str(),+ if is_spillover { "large" } else { initial },+ "{case}"+ );+ outcome.request.llm_request.model = None;+ assert_eq!(outcome.request.llm_request, request.llm_request);+ }+ let mut invalid = policy;+ invalid.min_waiting_requests = 0;+ assert!(invalid.validate().is_err());+ }+}diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex fe4d39fa3..f357e923e 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -20,6 +20,7 @@ pub use error::{DriverError, LibsyError, Result};mod algorithms;pub use algorithms::advisor_gate::{AdvisorGate, AdvisorGateConfig, GateTrigger};pub use algorithms::composite::{CompositeJudgeConfig, CompositeRouter, CompositeRouterConfig};+pub use algorithms::infra::InfraPolicy;pub use algorithms::llm_class::{CapabilityJudgeConfig, CustomClassifierConfig, CustomClassifierPolicy, DecisionJudgeConfig,LlmCapabilityConfig, LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig,diff --git a/crates/protocol/src/infra.rs b/crates/protocol/src/infra.rsnew file mode 100644index 000000000..13b6ef6e6--- /dev/null+++ b/crates/protocol/src/infra.rs@@ -0,0 +1,32 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Model-pool observations supplied by the serving integration.++use crate::ModelId;+use std::{collections::BTreeMap, time::Instant};++/// Observations keyed by model, without worker identities.+pub type InfraSignals = BTreeMap<ModelId, ModelInfraSignals>;++/// Pool observations and local request accounting.+#[derive(Clone, Debug, Default)]+pub struct ModelInfraSignals {+ /// None means unknown, never idle.+ pub load: Option<PoolLoad>,+ /// Requests still in flight through this host.+ pub local_inflight: u64,+}++/// Aggregated scheduler counts. The host is responsible for pool coverage.+#[derive(Clone, Debug)]+pub struct PoolLoad {+ /// Oldest underlying observation, expressed in the host's monotonic clock.+ pub sampled_at: Instant,+ /// Lowest queue depth among the pool's engines.+ pub min_waiting: u64,+ /// Highest queue depth among the pool's engines.+ pub max_waiting: u64,+ /// Total running requests across the pool.+ pub running: u64,+}diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rsindex b32dc6c43..0123e55c9 100644--- a/crates/protocol/src/lib.rs+++ b/crates/protocol/src/lib.rs@@ -10,6 +10,7 @@ pub mod codex_namespaces;pub mod decision;pub mod envelope;pub mod format;+pub mod infra;pub mod llm;pub mod metadata;pub mod model_id;@@ -20,6 +21,7 @@ pub use client::*;pub use decision::*;pub use envelope::*;pub use format::*;+pub use infra::*;pub use llm::*;pub use metadata::*;pub use model_id::*;diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex bbaad96c9..ca79eb453 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -107,6 +107,7 @@ struct RouteConfig {reasoning: Option<bool>,vision: Option<bool>,algorithm: AlgorithmSpec,+ infra: Option<libsy::InfraPolicy>,}struct TargetPromptPolicy {@@ -125,6 +126,7 @@ impl<'de> Deserialize<'de> for RouteConfig {let tool_calling = take_optional(&mut table, "tool_calling")?;let reasoning = take_optional(&mut table, "reasoning")?;let vision = take_optional(&mut table, "vision")?;+ let infra = take_optional(&mut table, "infra")?;let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table)).map_err(serde::de::Error::custom)?;Ok(Self {@@ -134,6 +136,7 @@ impl<'de> Deserialize<'de> for RouteConfig {reasoning,vision,algorithm,+ infra,})}}@@ -302,6 +305,19 @@ impl DeploymentConfig {.algorithm.build_with_runtime_models(route_name, &targets).map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;+ if let Some(infra) = &config.infra {+ if !matches!(&config.algorithm, AlgorithmSpec::LlmClassifier { config }+ if matches!(config.mode, None | Some(crate::ClassifierMode::Capability))+ && config.escalation.is_none())+ {+ return Err(RunnerError::configuration(format!(+ "route {route_name}: infra requires capability llm_classifier without escalation"+ )));+ }+ infra.validate().map_err(|error| {+ RunnerError::configuration_source(format!("route {route_name}: {error}"), error)+ })?;+ }if let AlgorithmSpec::Vgr { config: vgr } = &config.algorithm&& let Some(probe) =self.build_model_properties_probe(&vgr.efficient_target, &clients)@@ -328,7 +344,8 @@ impl DeploymentConfig {responses_auxiliary_target,decision_targets,models,- );+ )+ .with_infra(config.infra.clone());routes.push((config.id.clone(), route));}let runner = Runner::new(routes)@@ -1054,7 +1071,17 @@ efficient_target = "weak"#[test]fn public_runner_from_toml_builds_a_deployment() -> RunnerResult<()> {- let runner = Runner::from_toml(VALID_CONFIG)?;+ let config = format!(+ "{VALID_CONFIG}\n[routes.classifier.infra]\n\+ max_signal_age_ms = 1000\nmin_waiting_requests = 4\nmax_capable_inflight = 2"+ );+ let runner = Runner::from_toml(&config)?;+ assert!(+ Runner::from_toml(+ &config.replace("min_waiting_requests = 4", "min_waiting_requests = 0")+ )+ .is_err()+ );let classifier = runner.route("switchyard/classifier")diff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rsindex 36ff10841..48172c60a 100644--- a/crates/switchyard-runner/src/lib.rs+++ b/crates/switchyard-runner/src/lib.rs@@ -19,7 +19,7 @@ pub use algorithm::{};pub use failure::{RouteErrorKind, RouteErrorPhase, RouteErrorSummary, stream_error_summary};// Re-exported because `Route::new` takes it, so a host wiring routes does not need a libsy dep.-pub use libsy::RuntimeModels;+pub use libsy::{RoutingOutcome, RuntimeModels};pub use provider_key_redactor::ProviderKeyRedactor;pub use route::{AuxiliaryTarget, CallerAuthKind, ModelCapabilities, Route, RunOutput, RunnerError,diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rsindex 837233003..0d263a3b7 100644--- a/crates/switchyard-runner/src/route.rs+++ b/crates/switchyard-runner/src/route.rs@@ -120,6 +120,7 @@ pub struct Route {responses_auxiliary_target: Option<AuxiliaryTarget>,decision_targets: Vec<DecisionTarget>,models: Arc<RuntimeModels>,+ infra: Option<libsy::InfraPolicy>,}/// The selected model and untouched response produced by a route execution.@@ -150,6 +151,7 @@ impl Route {responses_auxiliary_target,decision_targets,models: Arc::new(models),+ infra: None,}}@@ -223,6 +225,28 @@ impl Route {.map_err(Into::into)}+ pub(crate) fn with_infra(mut self, infra: Option<libsy::InfraPolicy>) -> Self {+ self.infra = infra;+ self+ }++ /// Apply the optional queue policy to a completed task decision.+ ///+ /// The host must hold its in-flight accounting lock through this call and the+ /// increment for the selected model. Ordinary `decide` leaves task routing unchanged.+ pub fn apply_infra(+ &self,+ outcome: &mut RoutingOutcome,+ signals: &switchyard_protocol::InfraSignals,+ ) -> Result<bool, RunnerError> {+ match &self.infra {+ Some(policy) => policy+ .apply(outcome, &self.models, signals)+ .map_err(Into::into),+ None => Ok(false),+ }+ }+/// Routes with a host-owned client for routing-time LLM calls, such as judges.////// The host owns transport, credentials, and target request settings. Configureddiff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex 7f62a4fb8..93f20d66e 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -216,6 +216,25 @@ Every route takes the common keys below, plus the keys for its type.| `reasoning` | No | unset | When `false`, the server rejects reasoning controls with HTTP 400 before dispatch. Explicit `true` and unset values do not restrict requests. The server does not include this declaration in `GET /v1/models`. || `vision` | No | unset | Image-input support advertised in `GET /v1/models` under `data[].capabilities.vision`. When `false`, the server rejects images with HTTP 400 before dispatch, including images in tool results. Unset values appear as `null`. Explicit `true` and unset values do not restrict requests. Declare `true` only when every target the route can select accepts images. |+### Optional infrastructure check++A capability `llm_classifier` route without escalation may include `[routes.<name>.infra]`. Both tiers must have one model; each model may have+multiple workers. A host applies the policy explicitly with `Route::apply_infra`+after `Route::decide`, using periodically collected model-pool metrics and local in-flight+counts. Plain `decide` and `execute` retain task routing. Sub-agent routing is unchanged.++```toml+[routes.example.infra]+max_signal_age_ms = 3000+min_waiting_requests = 4+max_capable_inflight = 2+```++All keys are required. Limits must be positive. Missing or stale observations retain+the task decision. See the+[Dynamo PreProc example](../../examples/dynamo-preproc/README.md#task-classification-and-queue-based-routing)+for collection and capacity safeguards.+### `noop`Returns a buffered assistant response containing `OK` without calling andiff --git a/examples/dynamo-preproc/Cargo.toml b/examples/dynamo-preproc/Cargo.tomlindex b92dfb9e0..f4af4bad6 100644--- a/examples/dynamo-preproc/Cargo.toml+++ b/examples/dynamo-preproc/Cargo.toml@@ -26,6 +26,9 @@ switchyard-runner.workspace = trueswitchyard-protocol.workspace = trueswitchyard-translation.workspace = trueanyhow = "1"+reqwest.workspace = true+serde.workspace = true+toml = "1.1"async-stream.workspace = trueaxum = "0.8"futures-util.workspace = truediff --git a/examples/dynamo-preproc/Dockerfile b/examples/dynamo-preproc/Dockerfileindex d1f530974..a8c95839a 100644--- a/examples/dynamo-preproc/Dockerfile+++ b/examples/dynamo-preproc/Dockerfile@@ -19,6 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends cmake && rm -rfWORKDIR /appCOPY Cargo.toml Cargo.lock ./COPY crates ./crates+COPY examples/claude ./examples/claudeCOPY examples/dynamo-preproc ./examples/dynamo-preprocRUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=/app/target cargo build -p switchyard-preproc --release --locked && cp target/release/switchyard-preproc /app/switchyard-preprocdiff --git a/examples/dynamo-preproc/README.md b/examples/dynamo-preproc/README.mdindex caae6fbc9..1cd9103ac 100644--- a/examples/dynamo-preproc/README.md+++ b/examples/dynamo-preproc/README.md@@ -9,6 +9,10 @@ Run Switchyard as a separate ExtProc service behind agentgateway 1.0.0. The gatean OpenAI Chat Completions request to PreProc before matching its HTTPRoute. Switchyardchooses the model; the selected model's Dynamo EPP chooses the worker.+**Before you start:** you need an existing Kubernetes cluster with working GPU access+and `kubectl` configured to access it. This example does not create a cluster or+install GPU drivers.+```mermaidsequenceDiagramparticipant Client@@ -16,16 +20,13 @@ sequenceDiagramparticipant Proc as Switchyard PreProcparticipant EPP as Dynamo EPPparticipant Worker- Client->>Gateway: Chat completion (model: auto)- Gateway->>Proc: Request headers and body- Proc-->>Gateway: Selected model header and updated body- Gateway->>EPP: Route to selected InferencePool- EPP-->>Gateway: Worker endpoint++ Client->>Gateway: Request with model = auto+ Gateway->>Proc: Choose a model+ Proc-->>Gateway: Selected model and updated request+ Gateway->>EPP: Choose a worker in that model's pool+ EPP-->>Gateway: Selected workerGateway->>Worker: Forward request- Worker-->>Gateway: JSON or SSE response- Gateway->>Proc: Response headers and chunks- Proc-->>Gateway: Pass through unchanged- Gateway-->>Client: JSON or SSE response```This directory owns the service, tests and image build. The@@ -36,7 +37,7 @@ controller, model workers and EPPs are already installed.## Build the imageFrom the Switchyard repository root, use Docker to build and publish an image to a registry-your cluster can pull from:+your cluster can pull from. Replace the example registry and project below:```bashexport PREPROC_IMAGE=registry.example.com/your-project/switchyard-preproc:example@@ -60,15 +61,126 @@ Kustomization. Keep model IDs aligned with the HTTPRoutes and InferencePools.Policies must select a model without generating a response or rewriting the request.This example supports text and tool history with one PreProc replica. Routing state resets-on restart; Dynamo load and cache signals are not used.+on restart. The default StageRouter configuration does not use infrastructure signals.++## Task classification and queue-based routing++[task-classifier.toml](config/task-classifier.toml) classifies the task once per session.+Use a separate `X-Switchyard-Session-Id` per task. The classifier calls the large model+through the same gateway; `large_direct` skips classification for that explicit model ID.+Temperature zero and disabled Qwen thinking leave the judge's token budget for its verdict.++The optional `infra` rule can move an individual small-model request to the large model:++```text+every small worker has at least min_waiting_requests waiting+AND no large worker has requests waiting+AND max(total large-model running requests, local unfinished requests) < max_capable_inflight+AND both pools' samples are within max_signal_age_ms+```++Once the queue drains, requests use the original task choice again. EPP selects the+worker. This rule does not estimate waiting time or cache reuse. Removing `infra`+or `METRICS_CONFIG` disables the override; StageRouter is unaffected.++PreProc polls `sglang:num_queue_reqs` and `sglang:num_running_reqs` through the+Prometheus endpoint in [metrics.toml](config/metrics.toml). Select every engine in+each pool exactly once. Duplicate rank series or missing workers distort the counts;+this example does not detect those cases or check worker health.++Failed queries, missing aggregates or stale samples keep the task choice. Freshness+uses the oldest original scrape time, not the time of the Prometheus query.++Use one PreProc replica. Its capacity check and local increment are atomic; response+completion or cancellation releases the count. Metrics can lag other traffic, so+this is a routing guard, not a backend capacity reservation.++### Patch an existing Dynamo deployment++The [deployment patch](deploy/preproc-patch.yaml) enables this configuration on the+existing `switchyard-preproc` Deployment. It sets one replica and uses `Recreate`+updates to avoid overlapping processes with separate state. Updates interrupt service+and reset session state. No Dynamo repository changes are needed.++Enable SGLang's `--enable-metrics` and follow+[Dynamo's monitoring setup](https://docs.nvidia.com/dynamo/dev/kubernetes/operations/observability).+Dynamo's worker PodMonitor defaults to a five-second scrape interval;+set `max_signal_age_ms` to `15000` for that interval, or use one-second scrapes with+the supplied three-second limit.++Match the model IDs in both TOML files, including `large_direct`, to your HTTPRoutes.+The sample uses Qwen3-0.6B and Qwen3-1.7B; Dynamo's deployment example uses larger models.+Check the gateway and Prometheus URLs and the selectors' namespace. For QA, set+`fail_open = false` before creating the ConfigMap so classifier call errors surface.++Build and push this branch's image as described above, then run from the Switchyard root:++```bash+export EXAMPLE=examples/dynamo-preproc+export AGW_NAMESPACE=agentgateway-system+kubectl -n "$AGW_NAMESPACE" create configmap switchyard-task-config \+ --from-file=routes.toml="$EXAMPLE/config/task-classifier.toml" \+ --from-file=metrics.toml="$EXAMPLE/config/metrics.toml" \+ --dry-run=client -o yaml | kubectl apply -f -+kubectl -n "$AGW_NAMESPACE" patch deployment switchyard-preproc --type=strategic \+ --patch-file "$EXAMPLE/deploy/preproc-patch.yaml"+kubectl -n "$AGW_NAMESPACE" set image deployment/switchyard-preproc preproc="$PREPROC_IMAGE"+kubectl -n "$AGW_NAMESPACE" rollout status deployment/switchyard-preproc --timeout=180s+```++For later TOML edits, reapply the ConfigMap and restart `deployment/switchyard-preproc`+with `kubectl rollout restart`; the service loads configuration only at startup.++### Check the deployment++Wait for both model workers and EPPs to be ready, then forward the gateway port.+Keep this terminal open:++```bash+kubectl -n "$AGW_NAMESPACE" port-forward service/inference-gateway 8080:80+```++In another terminal, send a small task:++```bash+curl --fail-with-body -sS http://127.0.0.1:8080/v1/chat/completions \+ -H 'Content-Type: application/json' \+ -H 'X-Switchyard-Session-Id: qa-sort-1' \+ -d '{"model":"auto","messages":[{"role":"user","content":"Sort [3,1,2] in ascending order. Return only [1,2,3]. A validator compares your output to [1,2,3]."}],"max_tokens":32,"temperature":0,"chat_template_kwargs":{"enable_thinking":false}}'+```++Repeat with the same session ID. PreProc logs should show `consulting llm judge`+only for the first request. The response's `model` shows which model served it.+These tiny Qwen models are for integration checks; their classifications can vary.++```bash+kubectl -n "$AGW_NAMESPACE" logs deployment/switchyard-preproc --since=5m+```++For a manual queue check, use your deployment's load generator to send background+requests directly to the small pool. Keep one task session and verify:++- Quiet pool: small model, `small_pool_not_queued`.+- At least four waiting requests on every small worker, large pool idle: large model,+ `small_pool_queued`.+- Queue drained: small model again. A busy large pool gives `protect_capable_capacity`;+ missing or stale signals give `unknown_or_stale_signals` and keep the task choice.++Save the session ID, queue samples, response models and reason logs as QA evidence.## Concurrency and timeouts-agentgateway 1.0.0 sends responses through PreProc and cannot disable those phases.-Each request holds a stream slot until its response finishes. Set `MAX_ACTIVE_STREAMS` to change-the default of 16 per replica. Preprocessing also has a separate limit of eight concurrent-requests; reaching either limit returns HTTP 503. Raising the stream limit increases memory-and response-forwarding work.+agentgateway 1.0.0 sends responses through PreProc. Each request holds a slot until its+response finishes. `MAX_ACTIVE_STREAMS` defaults to 16 **per group**: routed requests+and configured passthrough requests have separate budgets, allowing up to 32 streams.+This lets judge calls re-enter PreProc while their parent requests wait. Explicit+model requests share the passthrough budget with judges.++At most eight requests can upload their bodies at once. The upload slot is released+before calling the classifier. A full upload or stream budget returns HTTP 503.++Preprocessing has a five-second total deadline, including the classifier call.+The example gives the judge two seconds. Metrics collection is outside this deadline.PreProc allows 120 seconds of response inactivity, resetting the deadline on each message.It also releases stream slots if output forwarding stalls for 120 seconds.diff --git a/examples/dynamo-preproc/config/metrics.toml b/examples/dynamo-preproc/config/metrics.tomlnew file mode 100644index 000000000..ea8bd75b6--- /dev/null+++ b/examples/dynamo-preproc/config/metrics.toml@@ -0,0 +1,16 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++# Service name from Dynamo's kube-prometheus-stack setup.+endpoint = "http://prometheus-kube-prometheus-prometheus.monitoring.svc.cluster.local:9090/api/v1/query"+poll_interval_ms = 1000+timeout_ms = 500++# Replace these with labels selecting every engine in the pool, without duplicate series.+[[models]]+model = "Qwen/Qwen3-0.6B"+selector = 'namespace="switchyard", model_name="Qwen/Qwen3-0.6B", endpoint="system"'++[[models]]+model = "Qwen/Qwen3-1.7B"+selector = 'namespace="switchyard", model_name="Qwen/Qwen3-1.7B", endpoint="system"'diff --git a/examples/dynamo-preproc/config/task-classifier.toml b/examples/dynamo-preproc/config/task-classifier.tomlnew file mode 100644index 000000000..3aaa01c0c--- /dev/null+++ b/examples/dynamo-preproc/config/task-classifier.toml@@ -0,0 +1,43 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++schema_version = 1++[llm_clients.dynamo]+format = "openai_chat"+base_url = "http://inference-gateway.agentgateway-system.svc.cluster.local/v1"+timeout_ms = 2000+max_retries = 0++[targets.qwen-small]+id = "Qwen/Qwen3-0.6B"+llm_client = "dynamo"++[targets.qwen-large]+id = "Qwen/Qwen3-1.7B"+llm_client = "dynamo"+extra_body = { temperature = 0, chat_template_kwargs = { enable_thinking = false } }++[routes.task]+id = "auto"+type = "llm_classifier"+mode = "capability"+classifier_target = "qwen-large"+weak_target = "qwen-small"+strong_target = "qwen-large"+classify_trigger = "new_session"+base_threshold = 0.5+fail_open = true+max_output_tokens = 256++# Remove this table for task-only routing. Tune limits for your deployment.+[routes.task.infra]+max_signal_age_ms = 3000+min_waiting_requests = 4+max_capable_inflight = 2++# The classifier calls this model through the same gateway without classifying again.+[routes.large_direct]+id = "Qwen/Qwen3-1.7B"+type = "passthrough"+target = "qwen-large"diff --git a/examples/dynamo-preproc/deploy/preproc-patch.yaml b/examples/dynamo-preproc/deploy/preproc-patch.yamlnew file mode 100644index 000000000..986c1bf84--- /dev/null+++ b/examples/dynamo-preproc/deploy/preproc-patch.yaml@@ -0,0 +1,22 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++# Strategic merge patch for Dynamo's existing switchyard-preproc Deployment.+spec:+ replicas: 1+ strategy:+ type: Recreate+ rollingUpdate: null+ template:+ spec:+ containers:+ - name: preproc+ env:+ - name: ROUTES_CONFIG+ value: /etc/switchyard/routes.toml+ - name: METRICS_CONFIG+ value: /etc/switchyard/metrics.toml+ volumes:+ - name: routing-config+ configMap:+ name: switchyard-task-configdiff --git a/examples/dynamo-preproc/src/main.rs b/examples/dynamo-preproc/src/main.rsindex 392c192c6..afc72f956 100644--- a/examples/dynamo-preproc/src/main.rs+++ b/examples/dynamo-preproc/src/main.rs@@ -13,6 +13,7 @@// See the License for the specific language governing permissions and// limitations under the License.+mod metrics;mod preprocessor;mod request;mod router;diff --git a/examples/dynamo-preproc/src/metrics.rs b/examples/dynamo-preproc/src/metrics.rsnew file mode 100644index 000000000..82811664f--- /dev/null+++ b/examples/dynamo-preproc/src/metrics.rs@@ -0,0 +1,317 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Prometheus pool observations and process-local request accounting.++use anyhow::{Context, ensure};+use serde::Deserialize;+use std::{+ collections::BTreeSet,+ path::Path,+ sync::{Arc, Mutex},+ time::{Duration, Instant, SystemTime, UNIX_EPOCH},+};+use switchyard_protocol::{InfraSignals, ModelId, PoolLoad};+use switchyard_runner::Route;++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct Config {+ endpoint: String,+ poll_interval_ms: u64,+ timeout_ms: u64,+ models: Vec<Model>,+}++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct Model {+ model: ModelId,+ /// Prometheus labels selecting the model pool.+ selector: String,+}++#[derive(Deserialize)]+struct Response {+ status: String,+ data: Vector,+ #[serde(default)]+ warnings: Vec<String>,+}++#[derive(Deserialize)]+struct Vector {+ result: Vec<Sample>,+}++#[derive(Deserialize)]+struct Sample {+ value: (f64, String),+}++struct Collector {+ config: Config,+ client: reqwest::Client,+}++impl Collector {+ fn load(path: &Path) -> anyhow::Result<Self> {+ let config: Config = toml::from_str(&std::fs::read_to_string(path)?)?;+ ensure!(+ config.poll_interval_ms > 0+ && config.timeout_ms > 0+ && config.timeout_ms <= config.poll_interval_ms+ && !config.models.is_empty(),+ "invalid collection limits"+ );+ let mut models = BTreeSet::new();+ for model in &config.models {+ ensure!(+ !model.selector.trim().is_empty() && models.insert(&model.model),+ "selectors must be nonempty and models unique"+ );+ }+ let url = reqwest::Url::parse(&config.endpoint)?;+ ensure!(+ matches!(url.scheme(), "http" | "https")+ && url.username().is_empty()+ && url.password().is_none(),+ "invalid Prometheus endpoint"+ );+ let client = reqwest::Client::builder()+ .timeout(Duration::from_millis(config.timeout_ms))+ .redirect(reqwest::redirect::Policy::none())+ .build()?;+ Ok(Self { config, client })+ }++ async fn query(&self, query: String, time: &str) -> anyhow::Result<f64> {+ let mut url = reqwest::Url::parse(&self.config.endpoint)?;+ url.query_pairs_mut()+ .extend_pairs([("query", query.as_str()), ("time", time)]);+ let response: Response = self+ .client+ .get(url)+ .send()+ .await?+ .error_for_status()?+ .json()+ .await?;+ ensure!(+ response.status == "success" && response.warnings.is_empty(),+ "incomplete Prometheus response"+ );+ let [sample] = response.data.result.as_slice() else {+ anyhow::bail!("expected one pool aggregate");+ };+ let value: f64 = sample.value.1.parse()?;+ ensure!(value.is_finite() && value >= 0.0, "invalid pool metric");+ Ok(value)+ }++ async fn collect(&self, model: &Model) -> anyhow::Result<PoolLoad> {+ let started = Instant::now();+ let time = SystemTime::now()+ .duration_since(UNIX_EPOCH)?+ .as_secs_f64()+ .to_string();+ let selector = &model.selector;+ let queue = format!("sglang:num_queue_reqs{{{selector}}}");+ let running = format!("sglang:num_running_reqs{{{selector}}}");+ // Aggregated samples carry query time; freshness needs the original scrape times.+ let (min_waiting, max_waiting, running, queue_age, running_age) = tokio::try_join!(+ self.query(format!("min({queue})"), &time),+ self.query(format!("max({queue})"), &time),+ self.query(format!("sum({running})"), &time),+ self.query(format!("time() - min(timestamp({queue}))"), &time),+ self.query(format!("time() - min(timestamp({running}))"), &time)+ )?;+ Ok(PoolLoad {+ sampled_at: started+ .checked_sub(Duration::try_from_secs_f64(queue_age.max(running_age))?)+ .context("invalid sample age")?,+ min_waiting: count(min_waiting)?,+ max_waiting: count(max_waiting)?,+ running: count(running)?,+ })+ }+}++fn count(value: f64) -> anyhow::Result<u64> {+ ensure!(+ value.is_finite() && value >= 0.0 && value.fract() == 0.0 && value < u64::MAX as f64,+ "invalid count"+ );+ Ok(value as u64)+}++pub struct Metrics {+ state: Arc<Mutex<InfraSignals>>,+ poller: tokio::task::JoinHandle<()>,+}++impl Metrics {+ pub fn load(path: &Path) -> anyhow::Result<Self> {+ let collector = Collector::load(path)?;+ let state = Arc::new(Mutex::new(InfraSignals::new()));+ let shared = state.clone();+ let poller = tokio::spawn(async move {+ let mut interval =+ tokio::time::interval(Duration::from_millis(collector.config.poll_interval_ms));+ interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);+ loop {+ interval.tick().await;+ for model in &collector.config.models {+ let load = collector.collect(model).await.inspect_err(|error| {+ tracing::warn!(model = %model.model, %error, "pool metrics unavailable");+ }).ok();+ // Clear failed observations without resetting in-flight counts.+ shared+ .lock()+ .unwrap_or_else(|p| p.into_inner())+ .entry(model.model.clone())+ .or_default()+ .load = load;+ }+ }+ });+ Ok(Self { state, poller })+ }++ pub fn track(+ &self,+ route: &Route,+ outcome: &mut switchyard_runner::RoutingOutcome,+ ) -> anyhow::Result<InFlight> {+ let mut signals = self.state.lock().unwrap_or_else(|p| p.into_inner());+ route.apply_infra(outcome, &signals)?;+ let model = outcome.selected_model_id()?.clone();+ signals.entry(model.clone()).or_default().local_inflight += 1;+ Ok(InFlight {+ state: self.state.clone(),+ model,+ })+ }+}++impl Drop for Metrics {+ fn drop(&mut self) {+ self.poller.abort();+ }+}++/// Lives until the ExtProc response stream finishes, errors or is dropped.+#[derive(Debug)]+pub struct InFlight {+ state: Arc<Mutex<InfraSignals>>,+ model: ModelId,+}++impl Drop for InFlight {+ fn drop(&mut self) {+ let mut signals = self.state.lock().unwrap_or_else(|p| p.into_inner());+ if let Some(counts) = signals.get_mut(&self.model) {+ counts.local_inflight -= 1;+ }+ }+}++#[cfg(test)]+impl Metrics {+ pub fn empty_for_test() -> Self {+ Self {+ state: Arc::new(Mutex::new(InfraSignals::new())),+ poller: tokio::spawn(std::future::pending()),+ }+ }++ pub fn inflight_for_test(&self) -> u64 {+ self.state+ .lock()+ .unwrap()+ .values()+ .map(|v| v.local_inflight)+ .sum()+ }+}+#[cfg(test)]+mod tests {+ use super::*;+ use axum::{Json, Router, extract::Query, routing::get};+ use serde_json::json;+ use std::{+ collections::BTreeMap,+ sync::atomic::{AtomicUsize, Ordering},+ };++ #[test]+ fn counts_require_finite_nonnegative_integers() {+ for value in [f64::NAN, -1.0, 1.5, f64::INFINITY, u64::MAX as f64] {+ assert!(count(value).is_err());+ }+ assert_eq!(count(0.0).unwrap(), 0);+ assert_eq!(count(8.0).unwrap(), 8);+ }++ #[tokio::test]+ async fn collection_preserves_missing_and_stale_signals() {+ let mode = Arc::new(AtomicUsize::new(0));+ let serving = mode.clone();+ let app = Router::new().route(+ "/api/v1/query",+ get(move |Query(query): Query<BTreeMap<String, String>>| {+ let mode = serving.load(Ordering::SeqCst);+ async move {+ assert!(query.contains_key("time"));+ let q = &query["query"];+ let value = match q.split('(').next().unwrap() {+ "time" if mode == 3 && q.contains("running") => "10",+ "time" => "0.01",+ "min" => "4",+ "max" => "5",+ "sum" => "2",+ _ => panic!("unexpected query: {q}"),+ };+ let result = if mode == 1 {+ vec![]+ } else {+ vec![json!({"metric":{}, "value":[1,value]})]+ };+ Json(+ json!({"status":"success","data":{"resultType":"vector","result":result},+ "warnings": if mode == 2 { vec!["partial response"] } else { vec![] }}),+ )+ }+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let server = tokio::spawn(async move {+ axum::serve(listener, app).await.unwrap();+ });+ let mut config: Config = toml::from_str(include_str!("../config/metrics.toml")).unwrap();+ config.endpoint = format!("http://{address}/api/v1/query");+ let collector = Collector {+ config,+ client: reqwest::Client::new(),+ };+ for case in 0..4 {+ mode.store(case, Ordering::SeqCst);+ let result = collector.collect(&collector.config.models[0]).await;+ if case == 1 || case == 2 {+ assert!(result.is_err());+ } else {+ let load = result.unwrap();+ assert_eq!(+ (load.min_waiting, load.max_waiting, load.running),+ (4, 5, 2)+ );+ assert!(+ load.sampled_at.elapsed()+ >= Duration::from_secs_f64(if case == 3 { 10.0 } else { 0.01 })+ );+ }+ }+ server.abort();+ }+}diff --git a/examples/dynamo-preproc/src/router.rs b/examples/dynamo-preproc/src/router.rsindex 5b5ddf173..6388b01d5 100644--- a/examples/dynamo-preproc/src/router.rs+++ b/examples/dynamo-preproc/src/router.rs@@ -32,29 +32,48 @@ pub const MODEL_HEADER: &str = "x-gateway-model-name";const SESSION_TTL: Duration = Duration::from_secs(60 * 60);type SessionIdentity = (String, String, Option<String>);+#[derive(serde::Deserialize)]+struct RequestModel {+ model: String,+}+pub struct Router {runner: Runner,sessions: Mutex<HashMap<SessionIdentity, Instant>>,+ pub(crate) metrics: Option<crate::metrics::Metrics>,}impl Router {pub fn load(routes: impl AsRef<Path>) -> Result<Self> {// SDK parse/build access would allow validation of decision-only options before construction.- Ok(Self::new(Runner::load(routes)?))+ let mut router = Self::new(Runner::load(routes)?);+ router.metrics = std::env::var("METRICS_CONFIG")+ .ok()+ .map(|path| crate::metrics::Metrics::load(Path::new(&path)))+ .transpose()?;+ Ok(router)}pub fn new(runner: Runner) -> Self {Self {runner,sessions: Mutex::new(HashMap::new()),+ metrics: None,}}+ pub fn is_passthrough(&self, body: &[u8]) -> bool {+ serde_json::from_slice::<RequestModel>(body)+ .ok()+ .and_then(|request| self.runner.route(&request.model))+ .is_some_and(|route| route.algorithm_name() == "passthrough")+ }+pub async fn decide(&self,body: &[u8],headers: &http::HeaderMap,- ) -> Result<(Vec<u8>, String)> {+ ) -> Result<(Vec<u8>, String, Option<crate::metrics::InFlight>)> {let started = Instant::now();let request = {let raw = serde_json::from_slice(body)?;@@ -103,10 +122,16 @@ impl Router {};let mut original_ir = request.llm_request.clone();original_ir.model = None;- let outcome = tokio::time::timeout(Duration::from_secs(1), route.decide(request))+ // The server's deadline includes body receipt and the classifier call.+ let mut outcome = route+ .decide(request).await- .map_err(|_| crate::server::Error::new(504, "SDK routing deadline exceeded"))?.map_err(|_| crate::server::Error::new(500, "SDK routing failed"))?;+ let inflight = self+ .metrics+ .as_ref()+ .map(|metrics| metrics.track(route, &mut outcome))+ .transpose()?;if is_session_final && let Some((identity, seen)) = admission {let mut sessions = self.sessions.lock().unwrap_or_else(|p| p.into_inner());// A newer request for this identity must retain its admission slot.@@ -129,7 +154,7 @@ impl Router {elapsed_us,"routing decision");- Ok((output, model.as_str().to_owned()))+ Ok((output, model.as_str().to_owned(), inflight))}}@@ -205,7 +230,7 @@ mod tests {"precise_decimal":0.12345678901234567890123456789,"custom\u005ffield":{ "nested": [18446744073709551617, {"text":"\u0061"}] }}"#;- let (output, model) = router()+ let (output, model, _) = router().decide(input, &http::HeaderMap::new()).await.unwrap();@@ -222,5 +247,14 @@ mod tests {assert_eq!(rewritten[&key].get(), value.get(), "changed {key}");}}+ let classifier =+ Router::new(Runner::from_toml(include_str!("../config/task-classifier.toml")).unwrap());+ let direct = replace_model(input, "Qwen/Qwen3-1.7B").unwrap();+ let (output, model, _) = classifier+ .decide(&direct, &http::HeaderMap::new())+ .await+ .unwrap();+ assert_eq!(model, "Qwen/Qwen3-1.7B");+ assert_eq!(output, direct);}}diff --git a/examples/dynamo-preproc/src/server.rs b/examples/dynamo-preproc/src/server.rsindex 294547f75..952a98d27 100644--- a/examples/dynamo-preproc/src/server.rs+++ b/examples/dynamo-preproc/src/server.rs@@ -22,7 +22,7 @@ use std::{use anyhow::{Context, ensure};use futures_util::{Stream, StreamExt};use tokio::{- sync::{Semaphore, mpsc},+ sync::{OwnedSemaphorePermit, Semaphore, mpsc},task::JoinSet,time::{Instant, timeout, timeout_at},};@@ -45,6 +45,7 @@ use crate::{const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);const RESPONSE_TIMEOUT: Duration = Duration::from_secs(120);const CHUNK_SIZE: usize = 64 * 1024;+const MAX_UPLOADS: usize = 8;#[derive(Debug, thiserror::Error)]#[error("{message}")]@@ -62,24 +63,29 @@ impl Error {}}+#[derive(Clone)]pub struct Server {router: Arc<Router>,requests: Arc<Semaphore>,streams: Arc<Semaphore>,+ direct_streams: Arc<Semaphore>,}struct Prepared {body: Vec<u8>,headers: ext::HeaderMutation,has_trailers: bool,+ inflight: Option<crate::metrics::InFlight>,+ stream_permit: OwnedSemaphorePermit,}impl Server {pub fn new(router: Arc<Router>, max_active_streams: usize) -> Self {Self {router,- requests: Arc::new(Semaphore::new(8)),+ requests: Arc::new(Semaphore::new(MAX_UPLOADS)),streams: Arc::new(Semaphore::new(max_active_streams)),+ direct_streams: Arc::new(Semaphore::new(max_active_streams)),}}@@ -88,8 +94,9 @@ impl Server {}async fn prepare(- router: &Router,+ &self,input: &mut Streaming<ext::ProcessingRequest>,+ request_permit: OwnedSemaphorePermit,) -> anyhow::Result<Prepared> {let first = input.message().await?.context("missing request headers")?;let Some(Input::RequestHeaders(first)) = first.request else {@@ -137,7 +144,18 @@ impl Server {_ => anyhow::bail!("expected request body"),}}- let (body, model) = router.decide(&body, &headers).await?;+ // Loopback judge calls need headroom while the parent holds its stream slot.+ let streams = if self.router.is_passthrough(&body) {+ &self.direct_streams+ } else {+ &self.streams+ };+ let stream_permit = streams+ .clone()+ .try_acquire_owned()+ .map_err(|_| Error::new(503, "processor stream limit reached"))?;+ drop(request_permit);+ let (body, model, inflight) = self.router.decide(&body, &headers).await?;if body.len() > MAX_BODY {return Err(Error::new(413, "rewritten request body exceeds 2 MiB").into());}@@ -149,6 +167,8 @@ impl Server {remove_headers,},has_trailers,+ inflight,+ stream_permit,})}}@@ -162,27 +182,16 @@ impl ExternalProcessor for Server {&self,request: Request<Streaming<ext::ProcessingRequest>>,) -> Result<Response<Self::ProcessStream>, Status> {- let stream_permit = self.streams.clone().try_acquire_owned();- let request_permit = stream_permit- .as_ref()- .ok()- .and_then(|_| self.requests.clone().try_acquire_owned().ok());- let router = self.router.clone();+ let request_permit = self.requests.clone().try_acquire_owned();+ let server = self.clone();let deadline = Instant::now() + REQUEST_TIMEOUT;let mut input = request.into_inner();let output = async_stream::stream! {- // These permits belong to the output stream, including while it is backpressured.- // Dropping that stream also cancels input reads and the routing future.- let Ok(_stream_permit) = stream_permit else {- yield Ok(reject(Error::new(503, "processor stream limit reached")));+ let Ok(request_permit) = request_permit else {+ yield Ok(reject(Error::new(503, "request upload limit reached")));return;};- let Some(request_permit) = request_permit else {- yield Ok(reject(Error::new(503, "preprocessor concurrency limit reached")));- return;- };- let result = timeout_at(deadline, Self::prepare(&router, &mut input)).await;- drop(request_permit);+ let result = timeout_at(deadline, server.prepare(&mut input, request_permit)).await;let prepared = match result {Ok(Ok(prepared)) => prepared,Ok(Err(error)) => {@@ -196,6 +205,9 @@ impl ExternalProcessor for Server {return;}};+ // Both guards live through response completion, cancellation and backpressure.+ let _stream_permit = prepared.stream_permit;+ let _inflight = prepared.inflight;let mut response_deadline = Instant::now() + RESPONSE_TIMEOUT;// v1.0 selects the route when this header reply arrives, before reading the body.yield Ok(reply(Output::RequestHeaders(ext::HeadersResponse {@@ -357,11 +369,17 @@ mod tests {type Client =ext::external_processor_client::ExternalProcessorClient<tonic::transport::Channel>;- async fn start(max_active_streams: usize) -> (Client, Arc<Semaphore>) {+ async fn start(+ max_active_streams: usize,+ config: &str,+ ) -> (Client, Arc<Semaphore>, Arc<Router>) {let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();let address = listener.local_addr().unwrap();- let runner = switchyard_runner::Runner::from_toml(include_str!("../config/routes.toml"));- let server = Server::new(Arc::new(Router::new(runner.unwrap())), max_active_streams);+ let runner = switchyard_runner::Runner::from_toml(config);+ let mut router = Router::new(runner.unwrap());+ router.metrics = Some(crate::metrics::Metrics::empty_for_test());+ let router = Arc::new(router);+ let server = Server::new(router.clone(), max_active_streams);let requests = server.requests.clone();tokio::spawn(async move {tonic::transport::Server::builder()@@ -379,6 +397,7 @@ mod tests {.unwrap(),),requests,+ router,)}@@ -421,6 +440,17 @@ mod tests {rx.message().await.unwrap().unwrap().response.unwrap()}+ async fn process_body(client: &mut Client, json: &[u8]) -> Streaming<ext::ProcessingResponse> {+ client+ .process(tokio_stream::iter([+ input(headers()),+ input(Input::RequestBody(body(json, true))),+ ]))+ .await+ .unwrap()+ .into_inner()+ }+fn streamed(reply: ext::BodyResponse) -> ext::StreamedBodyResponse {let Some(ext::body_mutation::Mutation::StreamedResponse(body)) =reply.response.unwrap().body_mutation.unwrap().mutation@@ -432,7 +462,7 @@ mod tests {#[tokio::test]async fn routes_upload_and_passes_response_through_trailers_to_eos() {- let (mut client, requests) = start(16).await;+ let (mut client, requests, router) = start(16, include_str!("../config/routes.toml")).await;let (tx, receive) = mpsc::channel(8);send(&tx, headers()).await;let receive = ReceiverStream::new(receive);@@ -463,7 +493,8 @@ mod tests {assert!(b.end_of_stream);let value: serde_json::Value = serde_json::from_slice(&b.body).unwrap();assert_eq!(value["model"], "Qwen/Qwen3-0.6B");- assert_eq!(requests.available_permits(), 8);+ assert_eq!(requests.available_permits(), MAX_UPLOADS);+ assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 1);send(&tx, Input::ResponseHeaders(ext::HttpHeaders::default())).await;assert!(matches!(next(&mut rx).await, Output::ResponseHeaders(_)));for (bytes, is_done) in [@@ -482,6 +513,28 @@ mod tests {}}assert!(rx.message().await.unwrap().is_none());+ assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 0);++ let (tx, receive) = mpsc::channel(8);+ send(&tx, headers()).await;+ send(&tx, Input::RequestBody(body(json, true))).await;+ let mut rx = client+ .process(ReceiverStream::new(receive))+ .await+ .unwrap()+ .into_inner();+ assert!(matches!(next(&mut rx).await, Output::RequestHeaders(_)));+ assert!(matches!(next(&mut rx).await, Output::RequestBody(_)));+ assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 1);+ drop(rx);+ drop(tx);+ tokio::time::timeout(Duration::from_secs(1), async {+ while router.metrics.as_ref().unwrap().inflight_for_test() != 0 {+ tokio::time::sleep(Duration::from_millis(5)).await;+ }+ })+ .await+ .unwrap();}#[tokio::test]@@ -493,7 +546,7 @@ mod tests {tokio::time::resume();}- let (mut client, _) = start(1).await;+ let (mut client, _, router) = start(1, include_str!("../config/routes.toml")).await;let (tx, receive) = mpsc::channel(8);send(&tx, headers()).await;let mut rx = client@@ -514,11 +567,11 @@ mod tests {send(&tx, Input::ResponseHeaders(ext::HttpHeaders::default())).await;assert!(matches!(next(&mut rx).await, Output::ResponseHeaders(_)));- let mut rejected = client- .process(tokio_stream::iter([input(headers())]))- .await- .unwrap()- .into_inner();+ let mut rejected = process_body(+ &mut client,+ br#"{"model":"auto","messages":[{"role":"user","content":"hello"}]}"#,+ )+ .await;let Output::ImmediateResponse(error) = next(&mut rejected).await else {panic!()};@@ -532,16 +585,18 @@ mod tests {};assert_eq!(streamed(b).body, b"data: hello\n\n");}+ assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 1);elapse(RESPONSE_TIMEOUT + Duration::from_secs(1)).await;assert_eq!(rx.message().await.unwrap_err().code(),tonic::Code::DeadlineExceeded);+ assert_eq!(router.metrics.as_ref().unwrap().inflight_for_test(), 0);}#[tokio::test]async fn rejects_cumulative_body_limit_before_routing_headers() {- let (mut client, _) = start(16).await;+ let (mut client, _, _) = start(16, include_str!("../config/routes.toml")).await;let messages = tokio_stream::iter([input(headers()),input(Input::RequestBody(body(&vec![b' '; MAX_BODY], false))),@@ -554,4 +609,58 @@ mod tests {assert_eq!(error.status.unwrap().code, 413);assert!(rx.message().await.unwrap().is_none());}++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn loopback_judge_runs_when_task_slots_are_full() {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let config = include_str!("../config/task-classifier.toml")+ .replace(+ "http://inference-gateway.agentgateway-system.svc.cluster.local/v1",+ &format!("http://{address}/v1"),+ )+ .replace("fail_open = true", "fail_open = false");+ let (client, _, _) = start(8, &config).await;+ let nested_client = client.clone();+ let barrier = Arc::new(tokio::sync::Barrier::new(8));+ let app = axum::Router::new().route("/v1/chat/completions", axum::routing::post(move || {+ let mut client = nested_client.clone();+ let barrier = barrier.clone();+ async move {+ // Hold all parent slots before their judge calls re-enter PreProc.+ barrier.wait().await;+ let json = br#"{"model":"Qwen/Qwen3-1.7B","messages":[{"role":"user","content":"judge"}]}"#;+ let mut rx = process_body(&mut client, json).await;+ assert!(matches!(next(&mut rx).await, Output::RequestHeaders(_)));+ assert!(matches!(next(&mut rx).await, Output::RequestBody(_)));+ axum::Json(serde_json::json!({+ "id":"judge", "object":"chat.completion", "created":0, "model":"Qwen/Qwen3-1.7B",+ "choices":[{"index":0,"message":{"role":"assistant","content":+ r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#+ },"finish_reason":"stop"}]+ }))+ }+ }));+ let http_task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ let mut tasks = JoinSet::new();+ for _ in 0..8 {+ let mut client = client.clone();+ tasks.spawn(async move {+ let json =+ br#"{"model":"auto","messages":[{"role":"user","content":"Sort [3,1,2]."}]}"#;+ let mut rx = process_body(&mut client, json).await;+ assert!(matches!(next(&mut rx).await, Output::RequestHeaders(_)));+ let Output::RequestBody(response) = next(&mut rx).await else {+ panic!("expected routed body")+ };+ let body: serde_json::Value =+ serde_json::from_slice(&streamed(response).body).unwrap();+ assert_eq!(body["model"], "Qwen/Qwen3-0.6B");+ });+ }+ while let Some(result) = tasks.join_next().await {+ result.unwrap();+ }+ http_task.abort();+ }}