Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(examples): add Dynamo Switchyard PreProc service (#919)
Ayush Agarwal79f2751f3e7f
diff --git a/Cargo.lock b/Cargo.lockindex 48b15ca63..5a6c710c7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -499,6 +499,17 @@ dependencies = ["serde",]+[[package]]+name = "envoy-types"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a4cda7cdc9e3fc567e65d34b8e710c97d8a1f5e3dfd4e9aa08976f5d5dd3e196"+dependencies = [+ "futures-core",+ "prost 0.13.5",+ "tonic",+]+[[package]]name = "equivalent"version = "1.0.2"@@ -855,6 +866,19 @@ dependencies = ["tower-service",]+[[package]]+name = "hyper-timeout"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"+dependencies = [+ "hyper",+ "hyper-util",+ "pin-project-lite",+ "tokio",+ "tower-service",+]+[[package]]name = "hyper-util"version = "0.1.20"@@ -872,7 +896,7 @@ dependencies = ["libc","percent-encoding","pin-project-lite",- "socket2",+ "socket2 0.6.5","tokio","tower-service","tracing",@@ -1420,7 +1444,7 @@ dependencies = ["opentelemetry-http","opentelemetry-proto","opentelemetry_sdk",- "prost",+ "prost 0.14.4","reqwest","thiserror 2.0.18",]@@ -1446,7 +1470,7 @@ checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"dependencies = ["opentelemetry","opentelemetry_sdk",- "prost",+ "prost 0.14.4",][[package]]@@ -1501,6 +1525,26 @@ version = "2.3.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"+[[package]]+name = "pin-project"+version = "1.1.13"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924"+dependencies = [+ "pin-project-internal",+]++[[package]]+name = "pin-project-internal"+version = "1.1.13"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]+[[package]]name = "pin-project-lite"version = "0.2.17"@@ -1585,6 +1629,16 @@ dependencies = ["thiserror 2.0.18",]+[[package]]+name = "prost"+version = "0.13.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"+dependencies = [+ "bytes",+ "prost-derive 0.13.5",+]+[[package]]name = "prost"version = "0.14.4"@@ -1592,7 +1646,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"dependencies = ["bytes",- "prost-derive",+ "prost-derive 0.14.4",+]++[[package]]+name = "prost-derive"+version = "0.13.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"+dependencies = [+ "anyhow",+ "itertools",+ "proc-macro2",+ "quote",+ "syn 2.0.119",][[package]]@@ -1723,7 +1790,7 @@ dependencies = ["quinn-udp","rustc-hash","rustls",- "socket2",+ "socket2 0.6.5","thiserror 2.0.18","tokio","tracing",@@ -1762,7 +1829,7 @@ dependencies = ["cfg_aliases","libc","once_cell",- "socket2",+ "socket2 0.6.5","tracing","windows-sys 0.61.2",]@@ -2270,6 +2337,16 @@ version = "1.15.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"+[[package]]+name = "socket2"+version = "0.5.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"+dependencies = [+ "libc",+ "windows-sys 0.52.0",+]+[[package]]name = "socket2"version = "0.6.5"@@ -2390,6 +2467,28 @@ dependencies = ["wiremock",]+[[package]]+name = "switchyard-preproc"+version = "0.1.0"+dependencies = [+ "anyhow",+ "async-stream",+ "axum",+ "envoy-types",+ "futures-util",+ "http",+ "serde_json",+ "switchyard-protocol",+ "switchyard-runner",+ "switchyard-translation",+ "thiserror 2.0.18",+ "tokio",+ "tokio-stream",+ "tonic",+ "tracing",+ "tracing-subscriber",+]+[[package]]name = "switchyard-protocol"version = "0.3.0"@@ -2664,7 +2763,7 @@ dependencies = ["parking_lot","pin-project-lite","signal-hook-registry",- "socket2",+ "socket2 0.6.5","tokio-macros","windows-sys 0.61.2",]@@ -2754,6 +2853,35 @@ version = "1.1.2+spec-1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2"+[[package]]+name = "tonic"+version = "0.13.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9"+dependencies = [+ "async-trait",+ "axum",+ "base64",+ "bytes",+ "h2",+ "http",+ "http-body",+ "http-body-util",+ "hyper",+ "hyper-timeout",+ "hyper-util",+ "percent-encoding",+ "pin-project",+ "prost 0.13.5",+ "socket2 0.5.10",+ "tokio",+ "tokio-stream",+ "tower",+ "tower-layer",+ "tower-service",+ "tracing",+]+[[package]]name = "tower"version = "0.5.3"@@ -2762,9 +2890,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"dependencies = ["futures-core","futures-util",+ "indexmap","pin-project-lite",+ "slab","sync_wrapper","tokio",+ "tokio-util","tower-layer","tower-service","tracing",diff --git a/Cargo.toml b/Cargo.tomlindex a3c8c9d02..a194b8023 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -4,6 +4,7 @@[workspace]resolver = "3"members = [+ "examples/dynamo-preproc","crates/libsy","crates/libsy-llm-client","crates/prefill-router",diff --git a/Dockerfile b/Dockerfileindex eb6815eea..e2d870eab 100644--- a/Dockerfile+++ b/Dockerfile@@ -10,6 +10,7 @@ COPY Cargo.toml Cargo.lock rust-toolchain.toml ./# .cargo/config.toml carries the workspace rustflags (target-cpu, force-frame-pointers).COPY .cargo ./.cargoCOPY crates ./crates+COPY examples/dynamo-preproc ./examples/dynamo-preprocRUN cargo build --locked --release -p switchyard-serverdiff --git a/README.md b/README.mdindex 59963b09c..c2b3efcb3 100644--- a/README.md+++ b/README.md@@ -23,6 +23,7 @@ Evaluate the complete agent, model pool, and routing configuration against your| Gateway | Start here | Current limits || --- | --- | --- || **LiteLLM** | [Run the Switchyard routing-plugin example](examples/litellm/README.md#quick-start-with-the-local-proxy) | Experimental and checkout-only. The example pins LiteLLM 1.102.0 and supports Stage routing based on request history, plus Random routing. It cannot service intermediate model calls required by classifier or escalation algorithms. |+| **Dynamo / agentgateway** | [Build the Switchyard PreProc example](examples/dynamo-preproc/README.md) | agentgateway 1.0.0, existing Dynamo model pools and EPPs. One PreProc replica; text and function-tool history only. || **NeMo Relay** | [Build and configure the native plugin](crates/switchyard-nemo-relay-plugin/README.md#build-from-source) | Requires Relay `>=0.8.0, <1.0.0`. The source-build path requires a Rust toolchain and Python 3. |Relay 0.8.x and 0.9.0 can lose upstream error status and details when the plugindiff --git a/benchmark/switchyard-server.Dockerfile b/benchmark/switchyard-server.Dockerfileindex 18185ba21..895f5a7ca 100644--- a/benchmark/switchyard-server.Dockerfile+++ b/benchmark/switchyard-server.Dockerfile@@ -17,6 +17,7 @@ ENV PATH="/opt/switchyard/.venv/bin:${PATH}" \COPY pyproject.toml uv.lock README.md Cargo.toml Cargo.lock ./COPY crates ./crates+COPY examples/dynamo-preproc ./examples/dynamo-preprocCOPY switchyard ./switchyardCOPY switchyard_rust ./switchyard_rustdiff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rsindex fff2ec46c..afe99c539 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs@@ -140,7 +140,7 @@ impl FormatCodec for OpenAiChatCodec {content: vec![ContentBlock::ToolResult(ToolResult {tool_call_id,content: vec![ContentBlock::Text { text }],- is_error: None,+ is_error: message.get("is_error").and_then(Value::as_bool),})],});continue;diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rsindex 3de0f5a47..6cd696397 100644--- a/crates/switchyard-translation/tests/request_translation.rs+++ b/crates/switchyard-translation/tests/request_translation.rs@@ -3311,6 +3311,30 @@ fn openai_stop_string_maps_to_anthropic_stop_sequences() -> TestResult {Ok(())}+#[test]+fn openai_tool_result_error_flag_survives_decoding() -> TestResult {+ let engine = TranslationEngine::default();+ for (flag, expected) in [+ (Some(json!(true)), Some(true)),+ (Some(json!(false)), Some(false)),+ (None, None),+ (Some(Value::Null), None),+ (Some(json!("true")), None),+ ] {+ let mut message = json!({"role": "tool", "tool_call_id": "call_1", "content": "result"});+ if let Some(flag) = flag {+ message["is_error"] = flag;+ }+ let body = json!({"model": "route", "messages": [message]});+ let decoded = engine.decode_request(WireFormat::OpenAiChat, &body, &normalized_policy())?;+ let ContentBlock::ToolResult(result) = &decoded.request.messages[0].content[0] else {+ panic!("expected tool result");+ };+ assert_eq!(result.is_error, expected);+ }+ Ok(())+}+// Verifies OpenAI tool results merge into Anthropic user tool-result content.#[test]fn openai_tool_results_are_merged_when_translating_to_anthropic() -> TestResult {diff --git a/examples/dynamo-preproc/Cargo.toml b/examples/dynamo-preproc/Cargo.tomlnew file mode 100644index 000000000..b92dfb9e0--- /dev/null+++ b/examples/dynamo-preproc/Cargo.toml@@ -0,0 +1,43 @@+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++[package]+name = "switchyard-preproc"+version = "0.1.0"+edition = "2024"+rust-version = "1.96.1"+license = "Apache-2.0"+publish = false++[dependencies]+switchyard-runner.workspace = true+switchyard-protocol.workspace = true+switchyard-translation.workspace = true+anyhow = "1"+async-stream.workspace = true+axum = "0.8"+futures-util.workspace = true+http.workspace = true+envoy-types = "=0.6.1"+serde_json = { workspace = true, features = ["raw_value"] }+tokio.workspace = true+tonic = "0.13.1"+thiserror.workspace = true+tracing.workspace = true+tracing-subscriber.workspace = true++[dev-dependencies]+tokio = { workspace = true, features = ["test-util"] }+tokio-stream = { version = "0.1", features = ["net"] }diff --git a/examples/dynamo-preproc/Dockerfile b/examples/dynamo-preproc/Dockerfilenew file mode 100644index 000000000..d1f530974--- /dev/null+++ b/examples/dynamo-preproc/Dockerfile@@ -0,0 +1,31 @@+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Build from the repository root to use the shared workspace lockfile.+FROM rust:1.96.1-bookworm@sha256:a339861ae23e9abb272cea45dfafde21760d2ce6577a70f8a926153677902663 AS build+RUN apt-get update && apt-get install -y --no-install-recommends cmake && rm -rf /var/lib/apt/lists/*+WORKDIR /app+COPY Cargo.toml Cargo.lock ./+COPY crates ./crates+COPY examples/dynamo-preproc ./examples/dynamo-preproc+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=/app/target cargo build -p switchyard-preproc --release --locked && cp target/release/switchyard-preproc /app/switchyard-preproc++FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*+COPY --from=build /app/switchyard-preproc /usr/local/bin/switchyard-preproc+ENV ROUTES_CONFIG=/etc/switchyard/routes.toml+USER 10001:10001+EXPOSE 9002 9003+ENTRYPOINT ["/usr/local/bin/switchyard-preproc"]diff --git a/examples/dynamo-preproc/README.md b/examples/dynamo-preproc/README.mdnew file mode 100644index 000000000..d0b4a1d04--- /dev/null+++ b/examples/dynamo-preproc/README.md@@ -0,0 +1,77 @@+<!--+SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+SPDX-License-Identifier: Apache-2.0+-->++# Switchyard PreProc for Dynamo++Run Switchyard as a separate ExtProc service behind agentgateway 1.0.0. The gateway sends+an OpenAI Chat Completions request to PreProc before matching its HTTPRoute. Switchyard+chooses the model; the selected model's Dynamo EPP chooses the worker.++```mermaid+sequenceDiagram+ participant Client+ participant Gateway as agentgateway+ participant Proc as Switchyard PreProc+ participant EPP as Dynamo EPP+ participant Worker+ Client->>Gateway: Chat completion (model: auto)+ Gateway->>Proc: Request headers and body+ Proc-->>Gateway: Selected model header and updated body+ Gateway->>EPP: Route to selected InferencePool+ EPP-->>Gateway: Worker endpoint+ Gateway->>Worker: Forward request+ Worker-->>Gateway: JSON or SSE response+ Gateway->>Proc: Response headers and chunks+ Proc-->>Gateway: Pass through unchanged+ Gateway-->>Client: JSON or SSE response+```++This directory owns the service, tests and image build. The+[Dynamo deployment example](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/vllm/deploy/gaie/switchyard)+owns the Kubernetes manifests and model-pool bindings. It assumes the cluster, gateway+controller, model workers and EPPs are already installed.++## Build the image++From the Switchyard repository root, use Docker to build and publish an image to a registry+your cluster can pull from:++```bash+export PREPROC_IMAGE=registry.example.com/your-project/switchyard-preproc:example+docker build -f examples/dynamo-preproc/Dockerfile -t "$PREPROC_IMAGE" .+docker push "$PREPROC_IMAGE"+```++Set that image in Dynamo's Kustomization and follow its deployment instructions.+The image uses the workspace Switchyard crates and precompiled Envoy bindings;+it uses the repository's shared lockfile and does not require a Dynamo checkout or a protobuf compiler.++## Configure routing++The supplied [routes.toml](config/routes.toml) uses StageRouter to choose between the two+Qwen models. Send `model: "auto"`; add `X-Switchyard-Session-Id` to retain routing state.+Send `X-Switchyard-Session-Final: true` on the final request to release its session admission+slot after a successful routing decision. Otherwise, idle slots expire after one hour.++To change routing, edit the TOML in the Dynamo deployment example and reapply its+Kustomization. Keep model IDs aligned with the HTTPRoutes and InferencePools.+Policies must select a model without generating a response or rewriting the request.++This example supports text and tool history with one PreProc replica. Routing state resets+on restart; Dynamo load and cache signals are not used.++## Concurrency and timeouts++agentgateway 1.0.0 sends responses through PreProc and cannot disable those phases.+Each request holds a stream slot until its response finishes. Set `MAX_ACTIVE_STREAMS` to change+the default of 16 per replica. Preprocessing also has a separate limit of eight concurrent+requests; reaching either limit returns HTTP 503. Raising the stream limit increases memory+and response-forwarding work.++PreProc allows 120 seconds of response inactivity, resetting the deadline on each message.+It also releases stream slots if output forwarding stalls for 120 seconds.+The Dynamo example's HTTPRoute uses a separate 120-second request deadline: in agentgateway+1.0.0 it bounds the wait for upstream response headers, measured from request start, rather+than the duration of an active response stream.diff --git a/examples/dynamo-preproc/config/routes.toml b/examples/dynamo-preproc/config/routes.tomlnew file mode 100644index 000000000..08f5374bb--- /dev/null+++ b/examples/dynamo-preproc/config/routes.toml@@ -0,0 +1,38 @@+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++schema_version = 1++# Native Switchyard runner configuration.+# This StageRouter policy makes no routing-time model calls.+[llm_clients.dynamo]+format = "openai_chat"+base_url = "http://switchyard-gateway/v1"++[targets.qwen-small]+id = "Qwen/Qwen3-0.6B"+llm_client = "dynamo"++[targets.qwen-large]+id = "Qwen/Qwen3-1.7B"+llm_client = "dynamo"++[routes.stage]+id = "auto"+type = "stage_router"+efficient_target = "qwen-small"+capable_target = "qwen-large"+picker = "efficient_first"+confidence_threshold = 0.5diff --git a/examples/dynamo-preproc/src/main.rs b/examples/dynamo-preproc/src/main.rsnew file mode 100644index 000000000..392c192c6--- /dev/null+++ b/examples/dynamo-preproc/src/main.rs@@ -0,0 +1,105 @@+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++mod preprocessor;+mod request;+mod router;+mod server;++use envoy_types::pb as proto;++use std::{+ sync::{+ Arc,+ atomic::{AtomicBool, Ordering},+ },+ time::Duration,+};++use anyhow::{Context, ensure};+use axum::{Router as HttpRouter, extract::State, http::StatusCode, routing::get};+use tokio::sync::{Semaphore, watch};+use tonic::transport::Server;++use crate::{preprocessor::MAX_BODY, router::Router};++async fn readiness(State(ready): State<Arc<AtomicBool>>) -> StatusCode {+ if ready.load(Ordering::Relaxed) {+ StatusCode::OK+ } else {+ StatusCode::SERVICE_UNAVAILABLE+ }+}++#[tokio::main]+async fn main() -> anyhow::Result<()> {+ tracing_subscriber::fmt()+ .with_env_filter(+ tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),+ )+ .init();+ let max_active_streams = std::env::var("MAX_ACTIVE_STREAMS")+ .unwrap_or_else(|_| "16".into())+ .parse::<usize>()+ .context("MAX_ACTIVE_STREAMS must be a positive integer")?;+ ensure!(+ (1..=Semaphore::MAX_PERMITS).contains(&max_active_streams),+ "MAX_ACTIVE_STREAMS must be between 1 and {}",+ Semaphore::MAX_PERMITS+ );+ let router = Arc::new(Router::load(+ std::env::var("ROUTES_CONFIG").unwrap_or_else(|_| "config/routes.toml".into()),+ )?);+ let ready = Arc::new(AtomicBool::new(true));+ let (shutdown, receiver) = watch::channel(false);+ let http = HttpRouter::new()+ .route("/healthz", get(|| async { StatusCode::OK }))+ .route("/readyz", get(readiness))+ .with_state(ready.clone());+ let listener = tokio::net::TcpListener::bind(+ std::env::var("ADMIN_ADDR").unwrap_or_else(|_| "0.0.0.0:9003".into()),+ )+ .await?;+ let mut http_shutdown = receiver.clone();+ let admin_task = tokio::spawn(async move {+ axum::serve(listener, http)+ .with_graceful_shutdown(async move {+ let _ = http_shutdown.changed().await;+ })+ .await+ });+ let processor = server::Server::new(router, max_active_streams)+ .into_service()+ .max_decoding_message_size(MAX_BODY + 65536)+ .max_encoding_message_size(MAX_BODY + 65536);+ let addr = std::env::var("GRPC_ADDR")+ .unwrap_or_else(|_| "0.0.0.0:9002".into())+ .parse()?;+ tracing::info!(%addr, max_active_streams, "configured decision-only preprocessor listening");+ let grpc = Server::builder()+ .http2_keepalive_interval(Some(Duration::from_secs(30)))+ .add_service(processor)+ .serve_with_shutdown(addr, async move {+ let mut terminate =+ tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())+ .expect("install SIGTERM handler");+ tokio::select! { _ = tokio::signal::ctrl_c() => {}, _ = terminate.recv() => {} }+ ready.store(false, Ordering::Relaxed);+ let _ = shutdown.send(true);+ });+ grpc.await?;+ admin_task.await??;+ Ok(())+}diff --git a/examples/dynamo-preproc/src/preprocessor.rs b/examples/dynamo-preproc/src/preprocessor.rsnew file mode 100644index 000000000..06e28ffbc--- /dev/null+++ b/examples/dynamo-preproc/src/preprocessor.rs@@ -0,0 +1,137 @@+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++use crate::{router::MODEL_HEADER, server::Error};++pub const MAX_BODY: usize = 2 * 1024 * 1024;++pub fn headers(input: &[(String, String)]) -> anyhow::Result<(http::HeaderMap, Vec<String>)> {+ let mut map = http::HeaderMap::new();+ let mut method = None;+ let mut path = None;+ let mut remove = [+ MODEL_HEADER,+ "x-gateway-destination-endpoint",+ "x-worker-instance-id",+ "x-prefill-instance-id",+ "x-prefiller-host-port",+ "x-dp-rank",+ "x-data-parallel-rank",+ "x-prefill-dp-rank",+ ]+ .into_iter()+ .map(str::to_owned)+ .collect::<Vec<_>>();+ for (key, value) in input {+ let key = key.to_ascii_lowercase();+ if key == ":method" {+ anyhow::ensure!(+ method.replace(value.to_owned()).is_none(),+ "duplicate method"+ );+ continue;+ }+ if key == ":path" {+ anyhow::ensure!(path.replace(value.to_owned()).is_none(), "duplicate path");+ continue;+ }+ if key.starts_with(':') {+ continue;+ }+ if key.starts_with("x-dynamo-") || key.starts_with("x-gateway-") {+ remove.push(key.clone());+ }+ let name = http::HeaderName::from_bytes(key.as_bytes())?;+ let unique = key.starts_with("x-switchyard-")+ || matches!(+ key.as_str(),+ "content-type" | "content-length" | "content-encoding"+ );+ anyhow::ensure!(+ !(unique && map.contains_key(&name)),+ "duplicate request header"+ );+ map.append(name, http::HeaderValue::from_str(value)?);+ }+ anyhow::ensure!(method.as_deref() == Some("POST"), "only POST is supported");+ anyhow::ensure!(+ path.as_deref().and_then(|p| p.split('?').next()) == Some("/v1/chat/completions"),+ "only /v1/chat/completions is supported"+ );+ anyhow::ensure!(+ map.get("content-type")+ .and_then(|v| v.to_str().ok())+ .is_some_and(|v| v+ .split(';')+ .next()+ .is_some_and(|mime| mime.trim().eq_ignore_ascii_case("application/json"))),+ "content-type must be application/json"+ );+ if map.contains_key("content-encoding") {+ return Err(Error::new(415, "compressed request bodies are unsupported").into());+ }+ if let Some(length) = map.get("content-length")+ && length.to_str()?.parse::<usize>()? > MAX_BODY+ {+ return Err(Error::new(413, "request body exceeds 2 MiB").into());+ }+ remove.sort();+ remove.dedup();+ Ok((map, remove))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn preserves_repeated_headers_but_rejects_ambiguous_routing_inputs() {+ let input = [+ (":method", "POST"),+ (":path", "/v1/chat/completions"),+ ("content-type", "application/json"),+ ("cookie", "a=1"),+ ("Cookie", "b=2"),+ ("x-dynamo-worker-id", "forged"),+ ]+ .map(|(k, v)| (k.to_owned(), v.to_owned()))+ .to_vec();+ let (map, remove) = headers(&input).unwrap();+ assert_eq!(+ map.get_all("cookie").iter().collect::<Vec<_>>(),+ ["a=1", "b=2"]+ );+ for key in [+ MODEL_HEADER,+ "x-gateway-destination-endpoint",+ "x-dynamo-worker-id",+ ] {+ assert!(remove.iter().any(|value| value == key));+ }+ for key in [+ "x-switchyard-session-id",+ "content-type",+ "content-length",+ "content-encoding",+ ] {+ let mut duplicate = input.clone();+ duplicate.extend([(key.into(), "a".into()), (key.into(), "b".into())]);+ assert_eq!(+ headers(&duplicate).unwrap_err().to_string(),+ "duplicate request header"+ );+ }+ }+}diff --git a/examples/dynamo-preproc/src/request.rs b/examples/dynamo-preproc/src/request.rsnew file mode 100644index 000000000..ae36f0bd9--- /dev/null+++ b/examples/dynamo-preproc/src/request.rs@@ -0,0 +1,152 @@+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++//! Deployment validation around Switchyard's OpenAI Chat decoder.+use anyhow::{Result, bail, ensure};+use serde_json::Value;+use switchyard_protocol::{Metadata, Request};+use switchyard_translation::{+ DeterministicIdPolicy, LossyConversionPolicy, PreservationPolicy, TranslationPolicy,+ codecs::{FormatCodec, openai_chat::OpenAiChatCodec},+};++fn validate_text(content: Option<&Value>) -> Result<()> {+ match content {+ None | Some(Value::Null | Value::String(_)) => Ok(()),+ Some(Value::Array(parts)) => {+ for part in parts {+ ensure!(+ part.get("type").and_then(Value::as_str) == Some("text")+ && part.get("text").is_some_and(Value::is_string),+ "only text content is supported by this model catalog"+ );+ }+ Ok(())+ }+ _ => bail!("message content must be text, text parts, or null"),+ }+}++pub fn decode(raw: &Value, headers: &http::HeaderMap) -> Result<Request> {+ validate_deployment(raw)?;+ let policy = TranslationPolicy {+ // Forward the original body; no IR-to-wire conversion or retained body copy is needed.+ preservation: PreservationPolicy::Disabled,+ lossy_conversion_policy: LossyConversionPolicy::Reject,+ deterministic_ids: DeterministicIdPolicy::Preserve,+ ..Default::default()+ };+ Ok(Request {+ llm_request: OpenAiChatCodec.decode_request(raw, &policy)?.request,+ raw_request: None,+ metadata: Some(Metadata::from_headers(headers)),+ })+}++fn validate_deployment(raw: &Value) -> Result<()> {+ if let Some(nvext) = raw.get("nvext").filter(|v| !v.is_null()) {+ ensure!(nvext.is_object(), "nvext must be an object");+ for reserved in [+ "token_data",+ "backend_instance_id",+ "prefill_worker_id",+ "decode_worker_id",+ "dp_rank",+ "prefill_dp_rank",+ "routing_constraints",+ "router",+ "use_raw_prompt",+ "metadata_upload",+ ] {+ ensure!(+ nvext.get(reserved).is_none_or(Value::is_null),+ "nvext contains a reserved worker or preprocessing control"+ );+ }+ }+ if let Some(messages) = raw.get("messages").and_then(Value::as_array) {+ for message in messages {+ validate_text(message.get("content"))?;+ }+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;+ use switchyard_protocol::{ContentBlock, ToolChoice};++ #[test]+ fn decodes_chat_controls_and_preserves_tool_error_flags() {+ let raw = json!({+ "model": "auto", "temperature": 0.2, "max_tokens": 64,+ "messages": [+ {"role": "system", "content": "Be concise"},+ {"role": "assistant", "tool_calls": [{"id": "c1", "type": "function",+ "function": {"name": "Bash", "arguments": "{\"command\":\"pytest\"}"}}]},+ {"role": "tool", "tool_call_id": "c1", "content": "failed", "is_error": true},+ {"role": "developer", "content": "Check the result"},+ {"role": "tool", "tool_call_id": "compacted", "content": "passed", "is_error": false}+ ],+ "tools": [{"type": "function", "function": {"name": "Bash", "parameters": {"type": "object"}}}],+ "tool_choice": "required"+ });+ let mut headers = http::HeaderMap::new();+ headers.insert("x-switchyard-session-id", "session-1".parse().unwrap());+ let request = decode(&raw, &headers).unwrap();+ let ir = request.llm_request;+ assert_eq!(ir.instructions.len(), 2);+ assert_eq!(ir.sampling.temperature, Some(0.2));+ assert_eq!(ir.output.max_output_tokens, Some(64));+ assert_eq!(ir.tool_choice, Some(ToolChoice::Required));+ assert_eq!(ir.tools[0].name, "Bash");+ let ContentBlock::ToolCall(call) = &ir.messages[0].content[0] else {+ panic!("missing tool call")+ };+ assert_eq!(call.arguments, json!({"command": "pytest"}));+ for (index, flag) in [(1, true), (2, false)] {+ let ContentBlock::ToolResult(result) = &ir.messages[index].content[0] else {+ panic!("missing tool result")+ };+ assert_eq!(result.is_error, Some(flag));+ }+ assert!(ir.preservation.requests.is_empty());+ assert!(request.raw_request.is_none());+ assert_eq!(+ request.metadata.unwrap().session_id.as_deref(),+ Some("session-1")+ );+ }++ #[test]+ fn rejects_reserved_controls_and_unsupported_media() {+ for extra in [+ json!({"nvext": {"backend_instance_id": 42}}),+ json!({"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.invalid/image"}}]}]}),+ ] {+ let mut raw =+ json!({"model": "auto", "messages": [{"role": "user", "content": "hello"}]});+ raw.as_object_mut()+ .unwrap()+ .extend(extra.as_object().unwrap().clone());+ assert!(+ decode(&raw, &http::HeaderMap::new()).is_err(),+ "accepted {raw}"+ );+ }+ }+}diff --git a/examples/dynamo-preproc/src/router.rs b/examples/dynamo-preproc/src/router.rsnew file mode 100644index 000000000..5b5ddf173--- /dev/null+++ b/examples/dynamo-preproc/src/router.rs@@ -0,0 +1,226 @@+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++use std::{+ collections::{BTreeMap, HashMap},+ path::Path,+ sync::Mutex,+ time::{Duration, Instant},+};++use anyhow::{Context, Result, ensure};+use serde_json::value::{RawValue, to_raw_value};+use switchyard_protocol::ModelId;+use switchyard_runner::Runner;++use crate::request;++pub const MODEL_HEADER: &str = "x-gateway-model-name";+// Match SDK 0.3's idle TTL; its retained state is reclaimed by an hourly sweep.+const SESSION_TTL: Duration = Duration::from_secs(60 * 60);+type SessionIdentity = (String, String, Option<String>);++pub struct Router {+ runner: Runner,+ sessions: Mutex<HashMap<SessionIdentity, Instant>>,+}++impl Router {+ pub fn load(routes: impl AsRef<Path>) -> Result<Self> {+ // SDK parse/build access would allow validation of decision-only options before construction.+ Ok(Self::new(Runner::load(routes)?))+ }++ pub fn new(runner: Runner) -> Self {+ Self {+ runner,+ sessions: Mutex::new(HashMap::new()),+ }+ }++ pub async fn decide(+ &self,+ body: &[u8],+ headers: &http::HeaderMap,+ ) -> Result<(Vec<u8>, String)> {+ let started = Instant::now();+ let request = {+ let raw = serde_json::from_slice(body)?;+ request::decode(&raw, headers)?+ };+ let input_model = request+ .llm_request+ .model+ .as_ref()+ .context("missing request model")?+ .as_str();+ let route_id = ModelId::from(input_model);+ let route = self+ .runner+ .route(route_id.as_str())+ .context("model is outside the configured catalog")?;+ let agent = request.metadata.as_ref().and_then(|m| m.agent_id.clone());+ ensure!(+ agent.as_ref().is_none_or(|id| id.len() <= 256),+ "agent id exceeds 256 bytes"+ );+ let is_session_final =+ request.metadata.as_ref().and_then(|m| m.session_final) == Some(true);+ let admission = if let Some(session) = request+ .metadata+ .as_ref()+ .and_then(|m| m.session_id.as_ref())+ {+ ensure!(session.len() <= 256, "session id exceeds 256 bytes");+ let identity = (route_id.as_str().to_owned(), session.clone(), agent);+ let mut sessions = self.sessions.lock().unwrap_or_else(|p| p.into_inner());+ let now = Instant::now();+ if !sessions.contains_key(&identity) && sessions.len() >= 4096 {+ sessions.retain(|_, seen| now.duration_since(*seen) < SESSION_TTL);+ }+ if !sessions.contains_key(&identity) && sessions.len() >= 4096 {+ return Err(+ crate::server::Error::new(503, "session identity capacity exceeded").into(),+ );+ }+ // The SDK can retain session state even when decide fails or is cancelled.+ sessions.insert(identity.clone(), now);+ Some((identity, now))+ } else {+ None+ };+ let mut original_ir = request.llm_request.clone();+ original_ir.model = None;+ let outcome = tokio::time::timeout(Duration::from_secs(1), route.decide(request))+ .await+ .map_err(|_| crate::server::Error::new(504, "SDK routing deadline exceeded"))?+ .map_err(|_| crate::server::Error::new(500, "SDK routing failed"))?;+ if is_session_final && let Some((identity, seen)) = admission {+ let mut sessions = self.sessions.lock().unwrap_or_else(|p| p.into_inner());+ // A newer request for this identity must retain its admission slot.+ if sessions.get(&identity) == Some(&seen) {+ sessions.remove(&identity);+ }+ }+ ensure!(outcome.response.is_none(), "routing-only contract violated");+ let model = outcome.selected_model_id()?.clone();+ let mut selected_ir = outcome.request.llm_request;+ selected_ir.model = None;+ ensure!(+ selected_ir == original_ir,+ "router unexpectedly rewrote request semantics"+ );+ let output = replace_model(body, model.as_str())?;+ let elapsed_us = started.elapsed().as_micros() as u64;+ tracing::info!(+ %model,+ elapsed_us,+ "routing decision"+ );+ Ok((output, model.as_str().to_owned()))+ }+}++fn replace_model(body: &[u8], model: &str) -> Result<Vec<u8>, serde_json::Error> {+ // Preserve original values, including precise numbers and provider-specific fields.+ let mut fields: BTreeMap<String, &RawValue> = serde_json::from_slice(body)?;+ let model = to_raw_value(model)?;+ fields.insert("model".to_owned(), &model);+ serde_json::to_vec(&fields)+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::{Value, json};++ fn router() -> Router {+ Router::new(Runner::from_toml(include_str!("../config/routes.toml")).unwrap())+ }+ fn neutral() -> Value {+ json!({"model":"auto", "messages":[{"role":"user", "content":"Say hello"}], "stream":true, "temperature":0.2, "custom_vendor_field":{"preserve":true}, "nvext":{"custom":"keep"}})+ }+ fn session(id: &str) -> http::HeaderMap {+ let mut h = http::HeaderMap::new();+ h.insert("x-switchyard-session-id", id.parse().unwrap());+ h+ }++ #[tokio::test]+ async fn session_capacity_recovers_after_idle_expiry() {+ let r = router();+ let quiet = serde_json::to_vec(&neutral()).unwrap();+ {+ let mut sessions = r.sessions.lock().unwrap();+ for i in 0..4096 {+ sessions.insert(("auto".into(), i.to_string(), None), Instant::now());+ }+ }+ let error = r.decide(&quiet, &session("new")).await.unwrap_err();+ assert_eq!(+ error+ .downcast_ref::<crate::server::Error>()+ .unwrap()+ .status_code,+ 503+ );+ let mut final_headers = session("0");+ final_headers.insert("x-switchyard-session-final", "true".parse().unwrap());+ r.decide(&quiet, &final_headers).await.unwrap();+ assert_eq!(r.sessions.lock().unwrap().len(), 4095);+ r.decide(&quiet, &session("new")).await.unwrap();+ assert_eq!(r.sessions.lock().unwrap().len(), 4096);+ {+ let mut sessions = r.sessions.lock().unwrap();+ for seen in sessions.values_mut() {+ *seen = Instant::now() - SESSION_TTL;+ }+ }+ r.decide(&quiet, &session("0")).await.unwrap();+ r.decide(&quiet, &session("new")).await.unwrap();+ let sessions = r.sessions.lock().unwrap();+ assert_eq!(sessions.len(), 2);+ assert!(sessions.contains_key(&("auto".into(), "0".into(), None)));+ assert!(sessions.contains_key(&("auto".into(), "new".into(), None)));+ }++ #[tokio::test]+ async fn preserves_original_values_including_precise_numbers_and_nested_fields() {+ let input = br#"{+ "mo\u0064el":"auto",+ "messages":[ {"role":"user", "content":"Hello"} ],+ "large_integer":18446744073709551617,+ "precise_decimal":0.12345678901234567890123456789,+ "custom\u005ffield":{ "nested": [18446744073709551617, {"text":"\u0061"}] }+ }"#;+ let (output, model) = router()+ .decide(input, &http::HeaderMap::new())+ .await+ .unwrap();+ assert_eq!(model, "Qwen/Qwen3-0.6B");+ let original: HashMap<String, &serde_json::value::RawValue> =+ serde_json::from_slice(input).unwrap();+ let rewritten: HashMap<String, &serde_json::value::RawValue> =+ serde_json::from_slice(&output).unwrap();+ assert_eq!(original.len(), rewritten.len());+ for (key, value) in original {+ if key == "model" {+ assert_eq!(rewritten[&key].get(), r#""Qwen/Qwen3-0.6B""#);+ } else {+ assert_eq!(rewritten[&key].get(), value.get(), "changed {key}");+ }+ }+ }+}diff --git a/examples/dynamo-preproc/src/server.rs b/examples/dynamo-preproc/src/server.rsnew file mode 100644index 000000000..294547f75--- /dev/null+++ b/examples/dynamo-preproc/src/server.rs@@ -0,0 +1,557 @@+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++use std::{+ pin::{Pin, pin},+ sync::Arc,+ time::Duration,+};++use anyhow::{Context, ensure};+use futures_util::{Stream, StreamExt};+use tokio::{+ sync::{Semaphore, mpsc},+ task::JoinSet,+ time::{Instant, timeout, timeout_at},+};+use tonic::{Request, Response, Status, Streaming};++use crate::{+ preprocessor::{MAX_BODY, headers},+ proto::envoy::{+ config::core::v3::{HeaderValue, HeaderValueOption},+ service::ext_proc::v3::{+ self as ext,+ external_processor_server::{ExternalProcessor, ExternalProcessorServer},+ processing_request::Request as Input,+ processing_response::Response as Output,+ },+ },+ router::{MODEL_HEADER, Router},+};++const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);+const RESPONSE_TIMEOUT: Duration = Duration::from_secs(120);+const CHUNK_SIZE: usize = 64 * 1024;++#[derive(Debug, thiserror::Error)]+#[error("{message}")]+pub struct Error {+ pub status_code: u16,+ pub message: String,+}++impl Error {+ pub fn new(status_code: u16, message: impl Into<String>) -> Self {+ Self {+ status_code,+ message: message.into(),+ }+ }+}++pub struct Server {+ router: Arc<Router>,+ requests: Arc<Semaphore>,+ streams: Arc<Semaphore>,+}++struct Prepared {+ body: Vec<u8>,+ headers: ext::HeaderMutation,+ has_trailers: bool,+}++impl Server {+ pub fn new(router: Arc<Router>, max_active_streams: usize) -> Self {+ Self {+ router,+ requests: Arc::new(Semaphore::new(8)),+ streams: Arc::new(Semaphore::new(max_active_streams)),+ }+ }++ pub fn into_service(self) -> ExternalProcessorServer<Self> {+ ExternalProcessorServer::new(self)+ }++ async fn prepare(+ router: &Router,+ input: &mut Streaming<ext::ProcessingRequest>,+ ) -> anyhow::Result<Prepared> {+ let first = input.message().await?.context("missing request headers")?;+ let Some(Input::RequestHeaders(first)) = first.request else {+ anyhow::bail!("request headers must arrive first");+ };+ ensure!(+ !first.end_of_stream,+ "chat completion requires a request body"+ );+ let values = first+ .headers+ .unwrap_or_default()+ .headers+ .into_iter()+ .map(|h| {+ let value = if h.raw_value.is_empty() {+ h.value+ } else {+ String::from_utf8(h.raw_value)?+ };+ Ok((h.key, value))+ })+ .collect::<anyhow::Result<Vec<_>>>()?;+ let (headers, mut remove_headers) = headers(&values)?;+ let mut body = Vec::new();+ let mut has_trailers = false;+ loop {+ let next = input.message().await?.context("incomplete request body")?;+ match next.request {+ Some(Input::RequestBody(chunk)) => {+ ensure!(+ !has_trailers || (chunk.body.is_empty() && chunk.end_of_stream),+ "expected body end after request trailers"+ );+ if chunk.body.len() > MAX_BODY - body.len() {+ return Err(Error::new(413, "request body exceeds 2 MiB").into());+ }+ body.extend_from_slice(&chunk.body);+ if chunk.end_of_stream {+ break;+ }+ }+ // Stock agentgateway 1.0 sends an empty body EOS after trailers.+ Some(Input::RequestTrailers(_)) if !has_trailers => has_trailers = true,+ _ => anyhow::bail!("expected request body"),+ }+ }+ let (body, model) = router.decide(&body, &headers).await?;+ if body.len() > MAX_BODY {+ return Err(Error::new(413, "rewritten request body exceeds 2 MiB").into());+ }+ remove_headers.push("content-length".into());+ Ok(Prepared {+ body,+ headers: ext::HeaderMutation {+ set_headers: vec![set_header(MODEL_HEADER, &model)],+ remove_headers,+ },+ has_trailers,+ })+ }+}++#[tonic::async_trait]+impl ExternalProcessor for Server {+ type ProcessStream =+ Pin<Box<dyn Stream<Item = Result<ext::ProcessingResponse, Status>> + Send>>;++ async fn process(+ &self,+ request: Request<Streaming<ext::ProcessingRequest>>,+ ) -> Result<Response<Self::ProcessStream>, Status> {+ let stream_permit = self.streams.clone().try_acquire_owned();+ let request_permit = stream_permit+ .as_ref()+ .ok()+ .and_then(|_| self.requests.clone().try_acquire_owned().ok());+ let router = self.router.clone();+ let deadline = Instant::now() + REQUEST_TIMEOUT;+ let mut input = request.into_inner();+ let output = async_stream::stream! {+ // These permits belong to the output stream, including while it is backpressured.+ // Dropping that stream also cancels input reads and the routing future.+ let Ok(_stream_permit) = stream_permit else {+ yield Ok(reject(Error::new(503, "processor stream limit reached")));+ return;+ };+ let Some(request_permit) = request_permit else {+ yield Ok(reject(Error::new(503, "preprocessor concurrency limit reached")));+ return;+ };+ let result = timeout_at(deadline, Self::prepare(&router, &mut input)).await;+ drop(request_permit);+ let prepared = match result {+ Ok(Ok(prepared)) => prepared,+ Ok(Err(error)) => {+ tracing::warn!(%error, "preprocessing rejected request");+ yield Ok(reject(error.downcast::<Error>()+ .unwrap_or_else(|e| Error::new(400, e.to_string()))));+ return;+ }+ Err(_) => {+ yield Ok(reject(Error::new(504, "preprocessing deadline exceeded")));+ return;+ }+ };+ let mut response_deadline = Instant::now() + RESPONSE_TIMEOUT;+ // v1.0 selects the route when this header reply arrives, before reading the body.+ yield Ok(reply(Output::RequestHeaders(ext::HeadersResponse {+ response: Some(ext::CommonResponse {+ header_mutation: Some(prepared.headers),+ ..Default::default()+ }),+ })));+ if prepared.has_trailers {+ yield Ok(reply(Output::RequestTrailers(ext::TrailersResponse::default())));+ }+ for (i, chunk) in prepared.body.chunks(CHUNK_SIZE).enumerate() {+ if Instant::now() >= response_deadline {+ yield Err(Status::deadline_exceeded("request output timed out"));+ return;+ }+ yield Ok(reply(Output::RequestBody(body_reply(+ chunk.to_vec(), (i + 1) * CHUNK_SIZE >= prepared.body.len(),+ ))));+ }+ response_deadline = Instant::now() + RESPONSE_TIMEOUT;+ let mut has_headers = false;+ let mut has_trailers = false;+ loop {+ let next = match timeout_at(response_deadline, input.message()).await {+ Ok(Ok(Some(next))) => next,+ Ok(Ok(None)) => {+ yield Err(Status::failed_precondition("incomplete response stream"));+ return;+ }+ Ok(Err(error)) => { yield Err(error); return; }+ Err(_) => {+ yield Err(Status::deadline_exceeded("response processing timed out"));+ return;+ }+ };+ response_deadline = Instant::now() + RESPONSE_TIMEOUT;+ let (output, is_done) = match next.request {+ Some(Input::ResponseHeaders(h)) if !has_headers => {+ has_headers = true;+ (Output::ResponseHeaders(ext::HeadersResponse {+ response: Some(ext::CommonResponse::default()),+ }), h.end_of_stream)+ }+ Some(Input::ResponseBody(b)) if has_headers+ && (!has_trailers || (b.body.is_empty() && b.end_of_stream)) => {+ let is_done = b.end_of_stream;+ (Output::ResponseBody(body_reply(b.body, is_done)), is_done)+ }+ Some(Input::ResponseTrailers(_)) if has_headers && !has_trailers => {+ has_trailers = true;+ // ACK trailers, then consume agentgateway's synthetic body EOS.+ (Output::ResponseTrailers(ext::TrailersResponse::default()), false)+ }+ _ => {+ yield Err(Status::invalid_argument("unexpected response processing phase"));+ return;+ }+ };+ yield Ok(reply(output));+ if is_done { return; }+ }+ };+ Ok(Response::new(Box::pin(with_output_timeout(output))))+ }+}++fn with_output_timeout(+ output: impl Stream<Item = Result<ext::ProcessingResponse, Status>> + Send + 'static,+) -> impl Stream<Item = Result<ext::ProcessingResponse, Status>> + Send {+ let (tx, mut rx) = mpsc::channel(1);+ let mut tasks = JoinSet::new();+ // Poll independently of tonic so a stalled reader cannot retain admission permits.+ tasks.spawn(async move {+ let mut output = pin!(output);+ while let Some(response) = output.next().await {+ if timeout(RESPONSE_TIMEOUT, tx.send(response))+ .await+ .map_err(|_| Status::deadline_exceeded("processor output stalled"))?+ .is_err()+ {+ break;+ }+ }+ Ok::<_, Status>(())+ });+ async_stream::stream! {+ // Owning the JoinSet also cancels the producer when this stream is dropped.+ while let Some(response) = rx.recv().await {+ yield response;+ }+ match tasks.join_next().await {+ Some(Ok(Err(error))) => yield Err(error),+ Some(Err(_)) => yield Err(Status::internal("processor task failed")),+ _ => {}+ }+ }+}++fn reply(response: Output) -> ext::ProcessingResponse {+ ext::ProcessingResponse {+ response: Some(response),+ ..Default::default()+ }+}++fn body_reply(body: Vec<u8>, is_end_of_stream: bool) -> ext::BodyResponse {+ ext::BodyResponse {+ response: Some(ext::CommonResponse {+ body_mutation: Some(ext::BodyMutation {+ mutation: Some(ext::body_mutation::Mutation::StreamedResponse(+ ext::StreamedBodyResponse {+ body,+ end_of_stream: is_end_of_stream,+ },+ )),+ }),+ ..Default::default()+ }),+ }+}++fn set_header(key: &str, value: &str) -> HeaderValueOption {+ HeaderValueOption {+ header: Some(HeaderValue {+ key: key.into(),+ value: String::new(),+ raw_value: value.as_bytes().to_vec(),+ }),+ append_action: 2,+ keep_empty_value: false,+ ..Default::default()+ }+}++fn reject(error: Error) -> ext::ProcessingResponse {+ reply(Output::ImmediateResponse(ext::ImmediateResponse {+ status: Some(crate::proto::envoy::r#type::v3::HttpStatus {+ code: i32::from(error.status_code),+ }),+ headers: Some(ext::HeaderMutation {+ set_headers: vec![set_header("content-type", "application/json")],+ remove_headers: vec![],+ }),+ body: serde_json::to_vec(&serde_json::json!({"error": {+ "message": error.message, "type": "switchyard_preproc_error"+ }}))+ .expect("JSON error object serializes"),+ details: "switchyard_preproc".into(),+ ..Default::default()+ }))+}++#[cfg(test)]+mod tests {+ use super::*;+ use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream};++ type Client =+ ext::external_processor_client::ExternalProcessorClient<tonic::transport::Channel>;++ async fn start(max_active_streams: usize) -> (Client, Arc<Semaphore>) {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let runner = switchyard_runner::Runner::from_toml(include_str!("../config/routes.toml"));+ let server = Server::new(Arc::new(Router::new(runner.unwrap())), max_active_streams);+ let requests = server.requests.clone();+ tokio::spawn(async move {+ tonic::transport::Server::builder()+ .add_service(server.into_service())+ .serve_with_incoming(TcpListenerStream::new(listener))+ .await+ .unwrap();+ });+ (+ Client::new(+ tonic::transport::Endpoint::from_shared(format!("http://{address}"))+ .unwrap()+ .connect()+ .await+ .unwrap(),+ ),+ requests,+ )+ }++ fn input(request: Input) -> ext::ProcessingRequest {+ ext::ProcessingRequest {+ request: Some(request),+ ..Default::default()+ }+ }++ fn body(bytes: &[u8], is_end_of_stream: bool) -> ext::HttpBody {+ ext::HttpBody {+ body: bytes.to_vec(),+ end_of_stream: is_end_of_stream,+ }+ }++ fn headers() -> Input {+ let headers = [+ (":method", "POST"),+ (":path", "/v1/chat/completions"),+ ("content-type", "application/json"),+ (MODEL_HEADER, "forged"),+ ]+ .into_iter()+ .map(|(k, v)| set_header(k, v).header.unwrap())+ .collect();+ Input::RequestHeaders(ext::HttpHeaders {+ headers: Some(crate::proto::envoy::config::core::v3::HeaderMap { headers }),+ end_of_stream: false,+ ..Default::default()+ })+ }++ async fn send(tx: &mpsc::Sender<ext::ProcessingRequest>, request: Input) {+ tx.send(input(request)).await.unwrap();+ }++ async fn next(rx: &mut Streaming<ext::ProcessingResponse>) -> Output {+ rx.message().await.unwrap().unwrap().response.unwrap()+ }++ fn streamed(reply: ext::BodyResponse) -> ext::StreamedBodyResponse {+ let Some(ext::body_mutation::Mutation::StreamedResponse(body)) =+ reply.response.unwrap().body_mutation.unwrap().mutation+ else {+ panic!()+ };+ body+ }++ #[tokio::test]+ async fn routes_upload_and_passes_response_through_trailers_to_eos() {+ let (mut client, requests) = start(16).await;+ let (tx, receive) = mpsc::channel(8);+ send(&tx, headers()).await;+ let receive = ReceiverStream::new(receive);+ let mut rx = client.process(receive).await.unwrap().into_inner();+ let json = br#"{"model":"auto","messages":[{"role":"user","content":"hello"}]}"#;+ for chunk in json.chunks(40) {+ send(&tx, Input::RequestBody(body(chunk, false))).await;+ }+ assert!(+ tokio::time::timeout(Duration::from_millis(30), rx.message())+ .await+ .is_err()+ );+ send(&tx, Input::RequestTrailers(ext::HttpTrailers::default())).await;+ send(&tx, Input::RequestBody(body(b"", true))).await;+ let Output::RequestHeaders(h) = next(&mut rx).await else {+ panic!()+ };+ let h = h.response.unwrap().header_mutation.unwrap();+ let selected = h.set_headers[0].header.as_ref().unwrap();+ assert_eq!(selected.raw_value, b"Qwen/Qwen3-0.6B");+ assert!(h.remove_headers.contains(&"content-length".to_owned()));+ assert!(matches!(next(&mut rx).await, Output::RequestTrailers(_)));+ let Output::RequestBody(b) = next(&mut rx).await else {+ panic!()+ };+ let b = streamed(b);+ assert!(b.end_of_stream);+ let value: serde_json::Value = serde_json::from_slice(&b.body).unwrap();+ assert_eq!(value["model"], "Qwen/Qwen3-0.6B");+ assert_eq!(requests.available_permits(), 8);+ send(&tx, Input::ResponseHeaders(ext::HttpHeaders::default())).await;+ assert!(matches!(next(&mut rx).await, Output::ResponseHeaders(_)));+ for (bytes, is_done) in [+ (b"data: hello\n\n".as_slice(), false),+ (b"".as_slice(), true),+ ] {+ send(&tx, Input::ResponseBody(body(bytes, is_done))).await;+ let Output::ResponseBody(b) = next(&mut rx).await else {+ panic!()+ };+ let b = streamed(b);+ assert_eq!((b.body.as_slice(), b.end_of_stream), (bytes, is_done));+ if !is_done {+ send(&tx, Input::ResponseTrailers(ext::HttpTrailers::default())).await;+ assert!(matches!(next(&mut rx).await, Output::ResponseTrailers(_)));+ }+ }+ assert!(rx.message().await.unwrap().is_none());+ }++ #[tokio::test]+ async fn response_timeout_tracks_inactivity() {+ async fn elapse(duration: Duration) {+ // Resume before socket I/O so idle network polls cannot auto-advance time.+ tokio::time::pause();+ tokio::time::advance(duration).await;+ tokio::time::resume();+ }++ let (mut client, _) = start(1).await;+ let (tx, receive) = mpsc::channel(8);+ send(&tx, headers()).await;+ let mut rx = client+ .process(ReceiverStream::new(receive))+ .await+ .unwrap()+ .into_inner();+ send(+ &tx,+ Input::RequestBody(body(+ br#"{"model":"auto","messages":[{"role":"user","content":"hello"}]}"#,+ true,+ )),+ )+ .await;+ assert!(matches!(next(&mut rx).await, Output::RequestHeaders(_)));+ assert!(matches!(next(&mut rx).await, Output::RequestBody(_)));+ send(&tx, Input::ResponseHeaders(ext::HttpHeaders::default())).await;+ assert!(matches!(next(&mut rx).await, Output::ResponseHeaders(_)));++ let mut rejected = client+ .process(tokio_stream::iter([input(headers())]))+ .await+ .unwrap()+ .into_inner();+ let Output::ImmediateResponse(error) = next(&mut rejected).await else {+ panic!()+ };+ assert_eq!(error.status.unwrap().code, 503);++ for _ in 0..3 {+ elapse(RESPONSE_TIMEOUT / 2).await;+ send(&tx, Input::ResponseBody(body(b"data: hello\n\n", false))).await;+ let Output::ResponseBody(b) = next(&mut rx).await else {+ panic!()+ };+ assert_eq!(streamed(b).body, b"data: hello\n\n");+ }+ elapse(RESPONSE_TIMEOUT + Duration::from_secs(1)).await;+ assert_eq!(+ rx.message().await.unwrap_err().code(),+ tonic::Code::DeadlineExceeded+ );+ }++ #[tokio::test]+ async fn rejects_cumulative_body_limit_before_routing_headers() {+ let (mut client, _) = start(16).await;+ let messages = tokio_stream::iter([+ input(headers()),+ input(Input::RequestBody(body(&vec![b' '; MAX_BODY], false))),+ input(Input::RequestBody(body(b" ", true))),+ ]);+ let mut rx = client.process(messages).await.unwrap().into_inner();+ let Output::ImmediateResponse(error) = next(&mut rx).await else {+ panic!()+ };+ assert_eq!(error.status.unwrap().code, 413);+ assert!(rx.message().await.unwrap().is_none());+ }+}