Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(examples): add Dynamo Switchyard PreProc service (#919)

Ayush Agarwal79f2751f3e7f
diff --git a/Cargo.lock b/Cargo.lock
index 48b15ca63..5a6c710c7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -499,6 +499,17 @@ dependencies = [
"serde",
]
+[[package]]
+name = "envoy-types"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4cda7cdc9e3fc567e65d34b8e710c97d8a1f5e3dfd4e9aa08976f5d5dd3e196"
+dependencies = [
+ "futures-core",
+ "prost 0.13.5",
+ "tonic",
+]
+
[[package]]
name = "equivalent"
version = "1.0.2"
@@ -855,6 +866,19 @@ dependencies = [
"tower-service",
]
+[[package]]
+name = "hyper-timeout"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
+dependencies = [
+ "hyper",
+ "hyper-util",
+ "pin-project-lite",
+ "tokio",
+ "tower-service",
+]
+
[[package]]
name = "hyper-util"
version = "0.1.20"
@@ -872,7 +896,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
- "socket2",
+ "socket2 0.6.5",
"tokio",
"tower-service",
"tracing",
@@ -1420,7 +1444,7 @@ dependencies = [
"opentelemetry-http",
"opentelemetry-proto",
"opentelemetry_sdk",
- "prost",
+ "prost 0.14.4",
"reqwest",
"thiserror 2.0.18",
]
@@ -1446,7 +1470,7 @@ checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"
dependencies = [
"opentelemetry",
"opentelemetry_sdk",
- "prost",
+ "prost 0.14.4",
]
[[package]]
@@ -1501,6 +1525,26 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+[[package]]
+name = "pin-project"
+version = "1.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -1585,6 +1629,16 @@ dependencies = [
"thiserror 2.0.18",
]
+[[package]]
+name = "prost"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
+dependencies = [
+ "bytes",
+ "prost-derive 0.13.5",
+]
+
[[package]]
name = "prost"
version = "0.14.4"
@@ -1592,7 +1646,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
dependencies = [
"bytes",
- "prost-derive",
+ "prost-derive 0.14.4",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
]
[[package]]
@@ -1723,7 +1790,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls",
- "socket2",
+ "socket2 0.6.5",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -1762,7 +1829,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
- "socket2",
+ "socket2 0.6.5",
"tracing",
"windows-sys 0.61.2",
]
@@ -2270,6 +2337,16 @@ version = "1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
+[[package]]
+name = "socket2"
+version = "0.5.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
[[package]]
name = "socket2"
version = "0.6.5"
@@ -2390,6 +2467,28 @@ dependencies = [
"wiremock",
]
+[[package]]
+name = "switchyard-preproc"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "async-stream",
+ "axum",
+ "envoy-types",
+ "futures-util",
+ "http",
+ "serde_json",
+ "switchyard-protocol",
+ "switchyard-runner",
+ "switchyard-translation",
+ "thiserror 2.0.18",
+ "tokio",
+ "tokio-stream",
+ "tonic",
+ "tracing",
+ "tracing-subscriber",
+]
+
[[package]]
name = "switchyard-protocol"
version = "0.3.0"
@@ -2664,7 +2763,7 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
- "socket2",
+ "socket2 0.6.5",
"tokio-macros",
"windows-sys 0.61.2",
]
@@ -2754,6 +2853,35 @@ version = "1.1.2+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2"
+[[package]]
+name = "tonic"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9"
+dependencies = [
+ "async-trait",
+ "axum",
+ "base64",
+ "bytes",
+ "h2",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-timeout",
+ "hyper-util",
+ "percent-encoding",
+ "pin-project",
+ "prost 0.13.5",
+ "socket2 0.5.10",
+ "tokio",
+ "tokio-stream",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
[[package]]
name = "tower"
version = "0.5.3"
@@ -2762,9 +2890,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
dependencies = [
"futures-core",
"futures-util",
+ "indexmap",
"pin-project-lite",
+ "slab",
"sync_wrapper",
"tokio",
+ "tokio-util",
"tower-layer",
"tower-service",
"tracing",
diff --git a/Cargo.toml b/Cargo.toml
index a3c8c9d02..a194b8023 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,6 +4,7 @@
[workspace]
resolver = "3"
members = [
+ "examples/dynamo-preproc",
"crates/libsy",
"crates/libsy-llm-client",
"crates/prefill-router",
diff --git a/Dockerfile b/Dockerfile
index eb6815eea..e2d870eab 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,6 +10,7 @@ COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
# .cargo/config.toml carries the workspace rustflags (target-cpu, force-frame-pointers).
COPY .cargo ./.cargo
COPY crates ./crates
+COPY examples/dynamo-preproc ./examples/dynamo-preproc
RUN cargo build --locked --release -p switchyard-server
diff --git a/README.md b/README.md
index 59963b09c..c2b3efcb3 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,7 @@ Evaluate the complete agent, model pool, and routing configuration against your
| Gateway | Start here | Current limits |
| --- | --- | --- |
| **LiteLLM** | [Run the Switchyard routing-plugin example](examples/litellm/README.md#quick-start-with-the-local-proxy) | Experimental and checkout-only. The example pins LiteLLM 1.102.0 and supports Stage routing based on request history, plus Random routing. It cannot service intermediate model calls required by classifier or escalation algorithms. |
+| **Dynamo / agentgateway** | [Build the Switchyard PreProc example](examples/dynamo-preproc/README.md) | agentgateway 1.0.0, existing Dynamo model pools and EPPs. One PreProc replica; text and function-tool history only. |
| **NeMo Relay** | [Build and configure the native plugin](crates/switchyard-nemo-relay-plugin/README.md#build-from-source) | Requires Relay `>=0.8.0, <1.0.0`. The source-build path requires a Rust toolchain and Python 3. |
Relay 0.8.x and 0.9.0 can lose upstream error status and details when the plugin
diff --git a/benchmark/switchyard-server.Dockerfile b/benchmark/switchyard-server.Dockerfile
index 18185ba21..895f5a7ca 100644
--- a/benchmark/switchyard-server.Dockerfile
+++ b/benchmark/switchyard-server.Dockerfile
@@ -17,6 +17,7 @@ ENV PATH="/opt/switchyard/.venv/bin:${PATH}" \
COPY pyproject.toml uv.lock README.md Cargo.toml Cargo.lock ./
COPY crates ./crates
+COPY examples/dynamo-preproc ./examples/dynamo-preproc
COPY switchyard ./switchyard
COPY switchyard_rust ./switchyard_rust
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index fff2ec46c..afe99c539 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -140,7 +140,7 @@ impl FormatCodec for OpenAiChatCodec {
content: vec![ContentBlock::ToolResult(ToolResult {
tool_call_id,
content: vec![ContentBlock::Text { text }],
- is_error: None,
+ is_error: message.get("is_error").and_then(Value::as_bool),
})],
});
continue;
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index 3de0f5a47..6cd696397 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -3311,6 +3311,30 @@ fn openai_stop_string_maps_to_anthropic_stop_sequences() -> TestResult {
Ok(())
}
+#[test]
+fn openai_tool_result_error_flag_survives_decoding() -> TestResult {
+ let engine = TranslationEngine::default();
+ for (flag, expected) in [
+ (Some(json!(true)), Some(true)),
+ (Some(json!(false)), Some(false)),
+ (None, None),
+ (Some(Value::Null), None),
+ (Some(json!("true")), None),
+ ] {
+ let mut message = json!({"role": "tool", "tool_call_id": "call_1", "content": "result"});
+ if let Some(flag) = flag {
+ message["is_error"] = flag;
+ }
+ let body = json!({"model": "route", "messages": [message]});
+ let decoded = engine.decode_request(WireFormat::OpenAiChat, &body, &normalized_policy())?;
+ let ContentBlock::ToolResult(result) = &decoded.request.messages[0].content[0] else {
+ panic!("expected tool result");
+ };
+ assert_eq!(result.is_error, expected);
+ }
+ Ok(())
+}
+
// Verifies OpenAI tool results merge into Anthropic user tool-result content.
#[test]
fn openai_tool_results_are_merged_when_translating_to_anthropic() -> TestResult {
diff --git a/examples/dynamo-preproc/Cargo.toml b/examples/dynamo-preproc/Cargo.toml
new file mode 100644
index 000000000..b92dfb9e0
--- /dev/null
+++ b/examples/dynamo-preproc/Cargo.toml
@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[package]
+name = "switchyard-preproc"
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.96.1"
+license = "Apache-2.0"
+publish = false
+
+[dependencies]
+switchyard-runner.workspace = true
+switchyard-protocol.workspace = true
+switchyard-translation.workspace = true
+anyhow = "1"
+async-stream.workspace = true
+axum = "0.8"
+futures-util.workspace = true
+http.workspace = true
+envoy-types = "=0.6.1"
+serde_json = { workspace = true, features = ["raw_value"] }
+tokio.workspace = true
+tonic = "0.13.1"
+thiserror.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["test-util"] }
+tokio-stream = { version = "0.1", features = ["net"] }
diff --git a/examples/dynamo-preproc/Dockerfile b/examples/dynamo-preproc/Dockerfile
new file mode 100644
index 000000000..d1f530974
--- /dev/null
+++ b/examples/dynamo-preproc/Dockerfile
@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build from the repository root to use the shared workspace lockfile.
+FROM rust:1.96.1-bookworm@sha256:a339861ae23e9abb272cea45dfafde21760d2ce6577a70f8a926153677902663 AS build
+RUN apt-get update && apt-get install -y --no-install-recommends cmake && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
+COPY Cargo.toml Cargo.lock ./
+COPY crates ./crates
+COPY examples/dynamo-preproc ./examples/dynamo-preproc
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=/app/target cargo build -p switchyard-preproc --release --locked && cp target/release/switchyard-preproc /app/switchyard-preproc
+
+FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
+COPY --from=build /app/switchyard-preproc /usr/local/bin/switchyard-preproc
+ENV ROUTES_CONFIG=/etc/switchyard/routes.toml
+USER 10001:10001
+EXPOSE 9002 9003
+ENTRYPOINT ["/usr/local/bin/switchyard-preproc"]
diff --git a/examples/dynamo-preproc/README.md b/examples/dynamo-preproc/README.md
new file mode 100644
index 000000000..d0b4a1d04
--- /dev/null
+++ b/examples/dynamo-preproc/README.md
@@ -0,0 +1,77 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Switchyard PreProc for Dynamo
+
+Run Switchyard as a separate ExtProc service behind agentgateway 1.0.0. The gateway sends
+an OpenAI Chat Completions request to PreProc before matching its HTTPRoute. Switchyard
+chooses the model; the selected model's Dynamo EPP chooses the worker.
+
+```mermaid
+sequenceDiagram
+ participant Client
+ participant Gateway as agentgateway
+ participant Proc as Switchyard PreProc
+ participant EPP as Dynamo EPP
+ participant Worker
+ Client->>Gateway: Chat completion (model: auto)
+ Gateway->>Proc: Request headers and body
+ Proc-->>Gateway: Selected model header and updated body
+ Gateway->>EPP: Route to selected InferencePool
+ EPP-->>Gateway: Worker endpoint
+ Gateway->>Worker: Forward request
+ Worker-->>Gateway: JSON or SSE response
+ Gateway->>Proc: Response headers and chunks
+ Proc-->>Gateway: Pass through unchanged
+ Gateway-->>Client: JSON or SSE response
+```
+
+This directory owns the service, tests and image build. The
+[Dynamo deployment example](https://github.com/ai-dynamo/dynamo/tree/main/examples/backends/vllm/deploy/gaie/switchyard)
+owns the Kubernetes manifests and model-pool bindings. It assumes the cluster, gateway
+controller, model workers and EPPs are already installed.
+
+## Build the image
+
+From the Switchyard repository root, use Docker to build and publish an image to a registry
+your cluster can pull from:
+
+```bash
+export PREPROC_IMAGE=registry.example.com/your-project/switchyard-preproc:example
+docker build -f examples/dynamo-preproc/Dockerfile -t "$PREPROC_IMAGE" .
+docker push "$PREPROC_IMAGE"
+```
+
+Set that image in Dynamo's Kustomization and follow its deployment instructions.
+The image uses the workspace Switchyard crates and precompiled Envoy bindings;
+it uses the repository's shared lockfile and does not require a Dynamo checkout or a protobuf compiler.
+
+## Configure routing
+
+The supplied [routes.toml](config/routes.toml) uses StageRouter to choose between the two
+Qwen models. Send `model: "auto"`; add `X-Switchyard-Session-Id` to retain routing state.
+Send `X-Switchyard-Session-Final: true` on the final request to release its session admission
+slot after a successful routing decision. Otherwise, idle slots expire after one hour.
+
+To change routing, edit the TOML in the Dynamo deployment example and reapply its
+Kustomization. Keep model IDs aligned with the HTTPRoutes and InferencePools.
+Policies must select a model without generating a response or rewriting the request.
+
+This example supports text and tool history with one PreProc replica. Routing state resets
+on restart; Dynamo load and cache signals are not used.
+
+## Concurrency and timeouts
+
+agentgateway 1.0.0 sends responses through PreProc and cannot disable those phases.
+Each request holds a stream slot until its response finishes. Set `MAX_ACTIVE_STREAMS` to change
+the default of 16 per replica. Preprocessing also has a separate limit of eight concurrent
+requests; reaching either limit returns HTTP 503. Raising the stream limit increases memory
+and response-forwarding work.
+
+PreProc allows 120 seconds of response inactivity, resetting the deadline on each message.
+It also releases stream slots if output forwarding stalls for 120 seconds.
+The Dynamo example's HTTPRoute uses a separate 120-second request deadline: in agentgateway
+1.0.0 it bounds the wait for upstream response headers, measured from request start, rather
+than the duration of an active response stream.
diff --git a/examples/dynamo-preproc/config/routes.toml b/examples/dynamo-preproc/config/routes.toml
new file mode 100644
index 000000000..08f5374bb
--- /dev/null
+++ b/examples/dynamo-preproc/config/routes.toml
@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+schema_version = 1
+
+# Native Switchyard runner configuration.
+# This StageRouter policy makes no routing-time model calls.
+[llm_clients.dynamo]
+format = "openai_chat"
+base_url = "http://switchyard-gateway/v1"
+
+[targets.qwen-small]
+id = "Qwen/Qwen3-0.6B"
+llm_client = "dynamo"
+
+[targets.qwen-large]
+id = "Qwen/Qwen3-1.7B"
+llm_client = "dynamo"
+
+[routes.stage]
+id = "auto"
+type = "stage_router"
+efficient_target = "qwen-small"
+capable_target = "qwen-large"
+picker = "efficient_first"
+confidence_threshold = 0.5
diff --git a/examples/dynamo-preproc/src/main.rs b/examples/dynamo-preproc/src/main.rs
new file mode 100644
index 000000000..392c192c6
--- /dev/null
+++ b/examples/dynamo-preproc/src/main.rs
@@ -0,0 +1,105 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod preprocessor;
+mod request;
+mod router;
+mod server;
+
+use envoy_types::pb as proto;
+
+use std::{
+ sync::{
+ Arc,
+ atomic::{AtomicBool, Ordering},
+ },
+ time::Duration,
+};
+
+use anyhow::{Context, ensure};
+use axum::{Router as HttpRouter, extract::State, http::StatusCode, routing::get};
+use tokio::sync::{Semaphore, watch};
+use tonic::transport::Server;
+
+use crate::{preprocessor::MAX_BODY, router::Router};
+
+async fn readiness(State(ready): State<Arc<AtomicBool>>) -> StatusCode {
+ if ready.load(Ordering::Relaxed) {
+ StatusCode::OK
+ } else {
+ StatusCode::SERVICE_UNAVAILABLE
+ }
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+ tracing_subscriber::fmt()
+ .with_env_filter(
+ tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()),
+ )
+ .init();
+ let max_active_streams = std::env::var("MAX_ACTIVE_STREAMS")
+ .unwrap_or_else(|_| "16".into())
+ .parse::<usize>()
+ .context("MAX_ACTIVE_STREAMS must be a positive integer")?;
+ ensure!(
+ (1..=Semaphore::MAX_PERMITS).contains(&max_active_streams),
+ "MAX_ACTIVE_STREAMS must be between 1 and {}",
+ Semaphore::MAX_PERMITS
+ );
+ let router = Arc::new(Router::load(
+ std::env::var("ROUTES_CONFIG").unwrap_or_else(|_| "config/routes.toml".into()),
+ )?);
+ let ready = Arc::new(AtomicBool::new(true));
+ let (shutdown, receiver) = watch::channel(false);
+ let http = HttpRouter::new()
+ .route("/healthz", get(|| async { StatusCode::OK }))
+ .route("/readyz", get(readiness))
+ .with_state(ready.clone());
+ let listener = tokio::net::TcpListener::bind(
+ std::env::var("ADMIN_ADDR").unwrap_or_else(|_| "0.0.0.0:9003".into()),
+ )
+ .await?;
+ let mut http_shutdown = receiver.clone();
+ let admin_task = tokio::spawn(async move {
+ axum::serve(listener, http)
+ .with_graceful_shutdown(async move {
+ let _ = http_shutdown.changed().await;
+ })
+ .await
+ });
+ let processor = server::Server::new(router, max_active_streams)
+ .into_service()
+ .max_decoding_message_size(MAX_BODY + 65536)
+ .max_encoding_message_size(MAX_BODY + 65536);
+ let addr = std::env::var("GRPC_ADDR")
+ .unwrap_or_else(|_| "0.0.0.0:9002".into())
+ .parse()?;
+ tracing::info!(%addr, max_active_streams, "configured decision-only preprocessor listening");
+ let grpc = Server::builder()
+ .http2_keepalive_interval(Some(Duration::from_secs(30)))
+ .add_service(processor)
+ .serve_with_shutdown(addr, async move {
+ let mut terminate =
+ tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
+ .expect("install SIGTERM handler");
+ tokio::select! { _ = tokio::signal::ctrl_c() => {}, _ = terminate.recv() => {} }
+ ready.store(false, Ordering::Relaxed);
+ let _ = shutdown.send(true);
+ });
+ grpc.await?;
+ admin_task.await??;
+ Ok(())
+}
diff --git a/examples/dynamo-preproc/src/preprocessor.rs b/examples/dynamo-preproc/src/preprocessor.rs
new file mode 100644
index 000000000..06e28ffbc
--- /dev/null
+++ b/examples/dynamo-preproc/src/preprocessor.rs
@@ -0,0 +1,137 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{router::MODEL_HEADER, server::Error};
+
+pub const MAX_BODY: usize = 2 * 1024 * 1024;
+
+pub fn headers(input: &[(String, String)]) -> anyhow::Result<(http::HeaderMap, Vec<String>)> {
+ let mut map = http::HeaderMap::new();
+ let mut method = None;
+ let mut path = None;
+ let mut remove = [
+ MODEL_HEADER,
+ "x-gateway-destination-endpoint",
+ "x-worker-instance-id",
+ "x-prefill-instance-id",
+ "x-prefiller-host-port",
+ "x-dp-rank",
+ "x-data-parallel-rank",
+ "x-prefill-dp-rank",
+ ]
+ .into_iter()
+ .map(str::to_owned)
+ .collect::<Vec<_>>();
+ for (key, value) in input {
+ let key = key.to_ascii_lowercase();
+ if key == ":method" {
+ anyhow::ensure!(
+ method.replace(value.to_owned()).is_none(),
+ "duplicate method"
+ );
+ continue;
+ }
+ if key == ":path" {
+ anyhow::ensure!(path.replace(value.to_owned()).is_none(), "duplicate path");
+ continue;
+ }
+ if key.starts_with(':') {
+ continue;
+ }
+ if key.starts_with("x-dynamo-") || key.starts_with("x-gateway-") {
+ remove.push(key.clone());
+ }
+ let name = http::HeaderName::from_bytes(key.as_bytes())?;
+ let unique = key.starts_with("x-switchyard-")
+ || matches!(
+ key.as_str(),
+ "content-type" | "content-length" | "content-encoding"
+ );
+ anyhow::ensure!(
+ !(unique && map.contains_key(&name)),
+ "duplicate request header"
+ );
+ map.append(name, http::HeaderValue::from_str(value)?);
+ }
+ anyhow::ensure!(method.as_deref() == Some("POST"), "only POST is supported");
+ anyhow::ensure!(
+ path.as_deref().and_then(|p| p.split('?').next()) == Some("/v1/chat/completions"),
+ "only /v1/chat/completions is supported"
+ );
+ anyhow::ensure!(
+ map.get("content-type")
+ .and_then(|v| v.to_str().ok())
+ .is_some_and(|v| v
+ .split(';')
+ .next()
+ .is_some_and(|mime| mime.trim().eq_ignore_ascii_case("application/json"))),
+ "content-type must be application/json"
+ );
+ if map.contains_key("content-encoding") {
+ return Err(Error::new(415, "compressed request bodies are unsupported").into());
+ }
+ if let Some(length) = map.get("content-length")
+ && length.to_str()?.parse::<usize>()? > MAX_BODY
+ {
+ return Err(Error::new(413, "request body exceeds 2 MiB").into());
+ }
+ remove.sort();
+ remove.dedup();
+ Ok((map, remove))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn preserves_repeated_headers_but_rejects_ambiguous_routing_inputs() {
+ let input = [
+ (":method", "POST"),
+ (":path", "/v1/chat/completions"),
+ ("content-type", "application/json"),
+ ("cookie", "a=1"),
+ ("Cookie", "b=2"),
+ ("x-dynamo-worker-id", "forged"),
+ ]
+ .map(|(k, v)| (k.to_owned(), v.to_owned()))
+ .to_vec();
+ let (map, remove) = headers(&input).unwrap();
+ assert_eq!(
+ map.get_all("cookie").iter().collect::<Vec<_>>(),
+ ["a=1", "b=2"]
+ );
+ for key in [
+ MODEL_HEADER,
+ "x-gateway-destination-endpoint",
+ "x-dynamo-worker-id",
+ ] {
+ assert!(remove.iter().any(|value| value == key));
+ }
+ for key in [
+ "x-switchyard-session-id",
+ "content-type",
+ "content-length",
+ "content-encoding",
+ ] {
+ let mut duplicate = input.clone();
+ duplicate.extend([(key.into(), "a".into()), (key.into(), "b".into())]);
+ assert_eq!(
+ headers(&duplicate).unwrap_err().to_string(),
+ "duplicate request header"
+ );
+ }
+ }
+}
diff --git a/examples/dynamo-preproc/src/request.rs b/examples/dynamo-preproc/src/request.rs
new file mode 100644
index 000000000..ae36f0bd9
--- /dev/null
+++ b/examples/dynamo-preproc/src/request.rs
@@ -0,0 +1,152 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Deployment validation around Switchyard's OpenAI Chat decoder.
+use anyhow::{Result, bail, ensure};
+use serde_json::Value;
+use switchyard_protocol::{Metadata, Request};
+use switchyard_translation::{
+ DeterministicIdPolicy, LossyConversionPolicy, PreservationPolicy, TranslationPolicy,
+ codecs::{FormatCodec, openai_chat::OpenAiChatCodec},
+};
+
+fn validate_text(content: Option<&Value>) -> Result<()> {
+ match content {
+ None | Some(Value::Null | Value::String(_)) => Ok(()),
+ Some(Value::Array(parts)) => {
+ for part in parts {
+ ensure!(
+ part.get("type").and_then(Value::as_str) == Some("text")
+ && part.get("text").is_some_and(Value::is_string),
+ "only text content is supported by this model catalog"
+ );
+ }
+ Ok(())
+ }
+ _ => bail!("message content must be text, text parts, or null"),
+ }
+}
+
+pub fn decode(raw: &Value, headers: &http::HeaderMap) -> Result<Request> {
+ validate_deployment(raw)?;
+ let policy = TranslationPolicy {
+ // Forward the original body; no IR-to-wire conversion or retained body copy is needed.
+ preservation: PreservationPolicy::Disabled,
+ lossy_conversion_policy: LossyConversionPolicy::Reject,
+ deterministic_ids: DeterministicIdPolicy::Preserve,
+ ..Default::default()
+ };
+ Ok(Request {
+ llm_request: OpenAiChatCodec.decode_request(raw, &policy)?.request,
+ raw_request: None,
+ metadata: Some(Metadata::from_headers(headers)),
+ })
+}
+
+fn validate_deployment(raw: &Value) -> Result<()> {
+ if let Some(nvext) = raw.get("nvext").filter(|v| !v.is_null()) {
+ ensure!(nvext.is_object(), "nvext must be an object");
+ for reserved in [
+ "token_data",
+ "backend_instance_id",
+ "prefill_worker_id",
+ "decode_worker_id",
+ "dp_rank",
+ "prefill_dp_rank",
+ "routing_constraints",
+ "router",
+ "use_raw_prompt",
+ "metadata_upload",
+ ] {
+ ensure!(
+ nvext.get(reserved).is_none_or(Value::is_null),
+ "nvext contains a reserved worker or preprocessing control"
+ );
+ }
+ }
+ if let Some(messages) = raw.get("messages").and_then(Value::as_array) {
+ for message in messages {
+ validate_text(message.get("content"))?;
+ }
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+ use switchyard_protocol::{ContentBlock, ToolChoice};
+
+ #[test]
+ fn decodes_chat_controls_and_preserves_tool_error_flags() {
+ let raw = json!({
+ "model": "auto", "temperature": 0.2, "max_tokens": 64,
+ "messages": [
+ {"role": "system", "content": "Be concise"},
+ {"role": "assistant", "tool_calls": [{"id": "c1", "type": "function",
+ "function": {"name": "Bash", "arguments": "{\"command\":\"pytest\"}"}}]},
+ {"role": "tool", "tool_call_id": "c1", "content": "failed", "is_error": true},
+ {"role": "developer", "content": "Check the result"},
+ {"role": "tool", "tool_call_id": "compacted", "content": "passed", "is_error": false}
+ ],
+ "tools": [{"type": "function", "function": {"name": "Bash", "parameters": {"type": "object"}}}],
+ "tool_choice": "required"
+ });
+ let mut headers = http::HeaderMap::new();
+ headers.insert("x-switchyard-session-id", "session-1".parse().unwrap());
+ let request = decode(&raw, &headers).unwrap();
+ let ir = request.llm_request;
+ assert_eq!(ir.instructions.len(), 2);
+ assert_eq!(ir.sampling.temperature, Some(0.2));
+ assert_eq!(ir.output.max_output_tokens, Some(64));
+ assert_eq!(ir.tool_choice, Some(ToolChoice::Required));
+ assert_eq!(ir.tools[0].name, "Bash");
+ let ContentBlock::ToolCall(call) = &ir.messages[0].content[0] else {
+ panic!("missing tool call")
+ };
+ assert_eq!(call.arguments, json!({"command": "pytest"}));
+ for (index, flag) in [(1, true), (2, false)] {
+ let ContentBlock::ToolResult(result) = &ir.messages[index].content[0] else {
+ panic!("missing tool result")
+ };
+ assert_eq!(result.is_error, Some(flag));
+ }
+ assert!(ir.preservation.requests.is_empty());
+ assert!(request.raw_request.is_none());
+ assert_eq!(
+ request.metadata.unwrap().session_id.as_deref(),
+ Some("session-1")
+ );
+ }
+
+ #[test]
+ fn rejects_reserved_controls_and_unsupported_media() {
+ for extra in [
+ json!({"nvext": {"backend_instance_id": 42}}),
+ json!({"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://example.invalid/image"}}]}]}),
+ ] {
+ let mut raw =
+ json!({"model": "auto", "messages": [{"role": "user", "content": "hello"}]});
+ raw.as_object_mut()
+ .unwrap()
+ .extend(extra.as_object().unwrap().clone());
+ assert!(
+ decode(&raw, &http::HeaderMap::new()).is_err(),
+ "accepted {raw}"
+ );
+ }
+ }
+}
diff --git a/examples/dynamo-preproc/src/router.rs b/examples/dynamo-preproc/src/router.rs
new file mode 100644
index 000000000..5b5ddf173
--- /dev/null
+++ b/examples/dynamo-preproc/src/router.rs
@@ -0,0 +1,226 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+ collections::{BTreeMap, HashMap},
+ path::Path,
+ sync::Mutex,
+ time::{Duration, Instant},
+};
+
+use anyhow::{Context, Result, ensure};
+use serde_json::value::{RawValue, to_raw_value};
+use switchyard_protocol::ModelId;
+use switchyard_runner::Runner;
+
+use crate::request;
+
+pub const MODEL_HEADER: &str = "x-gateway-model-name";
+// Match SDK 0.3's idle TTL; its retained state is reclaimed by an hourly sweep.
+const SESSION_TTL: Duration = Duration::from_secs(60 * 60);
+type SessionIdentity = (String, String, Option<String>);
+
+pub struct Router {
+ runner: Runner,
+ sessions: Mutex<HashMap<SessionIdentity, Instant>>,
+}
+
+impl Router {
+ pub fn load(routes: impl AsRef<Path>) -> Result<Self> {
+ // SDK parse/build access would allow validation of decision-only options before construction.
+ Ok(Self::new(Runner::load(routes)?))
+ }
+
+ pub fn new(runner: Runner) -> Self {
+ Self {
+ runner,
+ sessions: Mutex::new(HashMap::new()),
+ }
+ }
+
+ pub async fn decide(
+ &self,
+ body: &[u8],
+ headers: &http::HeaderMap,
+ ) -> Result<(Vec<u8>, String)> {
+ let started = Instant::now();
+ let request = {
+ let raw = serde_json::from_slice(body)?;
+ request::decode(&raw, headers)?
+ };
+ let input_model = request
+ .llm_request
+ .model
+ .as_ref()
+ .context("missing request model")?
+ .as_str();
+ let route_id = ModelId::from(input_model);
+ let route = self
+ .runner
+ .route(route_id.as_str())
+ .context("model is outside the configured catalog")?;
+ let agent = request.metadata.as_ref().and_then(|m| m.agent_id.clone());
+ ensure!(
+ agent.as_ref().is_none_or(|id| id.len() <= 256),
+ "agent id exceeds 256 bytes"
+ );
+ let is_session_final =
+ request.metadata.as_ref().and_then(|m| m.session_final) == Some(true);
+ let admission = if let Some(session) = request
+ .metadata
+ .as_ref()
+ .and_then(|m| m.session_id.as_ref())
+ {
+ ensure!(session.len() <= 256, "session id exceeds 256 bytes");
+ let identity = (route_id.as_str().to_owned(), session.clone(), agent);
+ let mut sessions = self.sessions.lock().unwrap_or_else(|p| p.into_inner());
+ let now = Instant::now();
+ if !sessions.contains_key(&identity) && sessions.len() >= 4096 {
+ sessions.retain(|_, seen| now.duration_since(*seen) < SESSION_TTL);
+ }
+ if !sessions.contains_key(&identity) && sessions.len() >= 4096 {
+ return Err(
+ crate::server::Error::new(503, "session identity capacity exceeded").into(),
+ );
+ }
+ // The SDK can retain session state even when decide fails or is cancelled.
+ sessions.insert(identity.clone(), now);
+ Some((identity, now))
+ } else {
+ None
+ };
+ let mut original_ir = request.llm_request.clone();
+ original_ir.model = None;
+ let outcome = tokio::time::timeout(Duration::from_secs(1), route.decide(request))
+ .await
+ .map_err(|_| crate::server::Error::new(504, "SDK routing deadline exceeded"))?
+ .map_err(|_| crate::server::Error::new(500, "SDK routing failed"))?;
+ if is_session_final && let Some((identity, seen)) = admission {
+ let mut sessions = self.sessions.lock().unwrap_or_else(|p| p.into_inner());
+ // A newer request for this identity must retain its admission slot.
+ if sessions.get(&identity) == Some(&seen) {
+ sessions.remove(&identity);
+ }
+ }
+ ensure!(outcome.response.is_none(), "routing-only contract violated");
+ let model = outcome.selected_model_id()?.clone();
+ let mut selected_ir = outcome.request.llm_request;
+ selected_ir.model = None;
+ ensure!(
+ selected_ir == original_ir,
+ "router unexpectedly rewrote request semantics"
+ );
+ let output = replace_model(body, model.as_str())?;
+ let elapsed_us = started.elapsed().as_micros() as u64;
+ tracing::info!(
+ %model,
+ elapsed_us,
+ "routing decision"
+ );
+ Ok((output, model.as_str().to_owned()))
+ }
+}
+
+fn replace_model(body: &[u8], model: &str) -> Result<Vec<u8>, serde_json::Error> {
+ // Preserve original values, including precise numbers and provider-specific fields.
+ let mut fields: BTreeMap<String, &RawValue> = serde_json::from_slice(body)?;
+ let model = to_raw_value(model)?;
+ fields.insert("model".to_owned(), &model);
+ serde_json::to_vec(&fields)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::{Value, json};
+
+ fn router() -> Router {
+ Router::new(Runner::from_toml(include_str!("../config/routes.toml")).unwrap())
+ }
+ fn neutral() -> Value {
+ json!({"model":"auto", "messages":[{"role":"user", "content":"Say hello"}], "stream":true, "temperature":0.2, "custom_vendor_field":{"preserve":true}, "nvext":{"custom":"keep"}})
+ }
+ fn session(id: &str) -> http::HeaderMap {
+ let mut h = http::HeaderMap::new();
+ h.insert("x-switchyard-session-id", id.parse().unwrap());
+ h
+ }
+
+ #[tokio::test]
+ async fn session_capacity_recovers_after_idle_expiry() {
+ let r = router();
+ let quiet = serde_json::to_vec(&neutral()).unwrap();
+ {
+ let mut sessions = r.sessions.lock().unwrap();
+ for i in 0..4096 {
+ sessions.insert(("auto".into(), i.to_string(), None), Instant::now());
+ }
+ }
+ let error = r.decide(&quiet, &session("new")).await.unwrap_err();
+ assert_eq!(
+ error
+ .downcast_ref::<crate::server::Error>()
+ .unwrap()
+ .status_code,
+ 503
+ );
+ let mut final_headers = session("0");
+ final_headers.insert("x-switchyard-session-final", "true".parse().unwrap());
+ r.decide(&quiet, &final_headers).await.unwrap();
+ assert_eq!(r.sessions.lock().unwrap().len(), 4095);
+ r.decide(&quiet, &session("new")).await.unwrap();
+ assert_eq!(r.sessions.lock().unwrap().len(), 4096);
+ {
+ let mut sessions = r.sessions.lock().unwrap();
+ for seen in sessions.values_mut() {
+ *seen = Instant::now() - SESSION_TTL;
+ }
+ }
+ r.decide(&quiet, &session("0")).await.unwrap();
+ r.decide(&quiet, &session("new")).await.unwrap();
+ let sessions = r.sessions.lock().unwrap();
+ assert_eq!(sessions.len(), 2);
+ assert!(sessions.contains_key(&("auto".into(), "0".into(), None)));
+ assert!(sessions.contains_key(&("auto".into(), "new".into(), None)));
+ }
+
+ #[tokio::test]
+ async fn preserves_original_values_including_precise_numbers_and_nested_fields() {
+ let input = br#"{
+ "mo\u0064el":"auto",
+ "messages":[ {"role":"user", "content":"Hello"} ],
+ "large_integer":18446744073709551617,
+ "precise_decimal":0.12345678901234567890123456789,
+ "custom\u005ffield":{ "nested": [18446744073709551617, {"text":"\u0061"}] }
+ }"#;
+ let (output, model) = router()
+ .decide(input, &http::HeaderMap::new())
+ .await
+ .unwrap();
+ assert_eq!(model, "Qwen/Qwen3-0.6B");
+ let original: HashMap<String, &serde_json::value::RawValue> =
+ serde_json::from_slice(input).unwrap();
+ let rewritten: HashMap<String, &serde_json::value::RawValue> =
+ serde_json::from_slice(&output).unwrap();
+ assert_eq!(original.len(), rewritten.len());
+ for (key, value) in original {
+ if key == "model" {
+ assert_eq!(rewritten[&key].get(), r#""Qwen/Qwen3-0.6B""#);
+ } else {
+ assert_eq!(rewritten[&key].get(), value.get(), "changed {key}");
+ }
+ }
+ }
+}
diff --git a/examples/dynamo-preproc/src/server.rs b/examples/dynamo-preproc/src/server.rs
new file mode 100644
index 000000000..294547f75
--- /dev/null
+++ b/examples/dynamo-preproc/src/server.rs
@@ -0,0 +1,557 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+ pin::{Pin, pin},
+ sync::Arc,
+ time::Duration,
+};
+
+use anyhow::{Context, ensure};
+use futures_util::{Stream, StreamExt};
+use tokio::{
+ sync::{Semaphore, mpsc},
+ task::JoinSet,
+ time::{Instant, timeout, timeout_at},
+};
+use tonic::{Request, Response, Status, Streaming};
+
+use crate::{
+ preprocessor::{MAX_BODY, headers},
+ proto::envoy::{
+ config::core::v3::{HeaderValue, HeaderValueOption},
+ service::ext_proc::v3::{
+ self as ext,
+ external_processor_server::{ExternalProcessor, ExternalProcessorServer},
+ processing_request::Request as Input,
+ processing_response::Response as Output,
+ },
+ },
+ router::{MODEL_HEADER, Router},
+};
+
+const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
+const RESPONSE_TIMEOUT: Duration = Duration::from_secs(120);
+const CHUNK_SIZE: usize = 64 * 1024;
+
+#[derive(Debug, thiserror::Error)]
+#[error("{message}")]
+pub struct Error {
+ pub status_code: u16,
+ pub message: String,
+}
+
+impl Error {
+ pub fn new(status_code: u16, message: impl Into<String>) -> Self {
+ Self {
+ status_code,
+ message: message.into(),
+ }
+ }
+}
+
+pub struct Server {
+ router: Arc<Router>,
+ requests: Arc<Semaphore>,
+ streams: Arc<Semaphore>,
+}
+
+struct Prepared {
+ body: Vec<u8>,
+ headers: ext::HeaderMutation,
+ has_trailers: bool,
+}
+
+impl Server {
+ pub fn new(router: Arc<Router>, max_active_streams: usize) -> Self {
+ Self {
+ router,
+ requests: Arc::new(Semaphore::new(8)),
+ streams: Arc::new(Semaphore::new(max_active_streams)),
+ }
+ }
+
+ pub fn into_service(self) -> ExternalProcessorServer<Self> {
+ ExternalProcessorServer::new(self)
+ }
+
+ async fn prepare(
+ router: &Router,
+ input: &mut Streaming<ext::ProcessingRequest>,
+ ) -> anyhow::Result<Prepared> {
+ let first = input.message().await?.context("missing request headers")?;
+ let Some(Input::RequestHeaders(first)) = first.request else {
+ anyhow::bail!("request headers must arrive first");
+ };
+ ensure!(
+ !first.end_of_stream,
+ "chat completion requires a request body"
+ );
+ let values = first
+ .headers
+ .unwrap_or_default()
+ .headers
+ .into_iter()
+ .map(|h| {
+ let value = if h.raw_value.is_empty() {
+ h.value
+ } else {
+ String::from_utf8(h.raw_value)?
+ };
+ Ok((h.key, value))
+ })
+ .collect::<anyhow::Result<Vec<_>>>()?;
+ let (headers, mut remove_headers) = headers(&values)?;
+ let mut body = Vec::new();
+ let mut has_trailers = false;
+ loop {
+ let next = input.message().await?.context("incomplete request body")?;
+ match next.request {
+ Some(Input::RequestBody(chunk)) => {
+ ensure!(
+ !has_trailers || (chunk.body.is_empty() && chunk.end_of_stream),
+ "expected body end after request trailers"
+ );
+ if chunk.body.len() > MAX_BODY - body.len() {
+ return Err(Error::new(413, "request body exceeds 2 MiB").into());
+ }
+ body.extend_from_slice(&chunk.body);
+ if chunk.end_of_stream {
+ break;
+ }
+ }
+ // Stock agentgateway 1.0 sends an empty body EOS after trailers.
+ Some(Input::RequestTrailers(_)) if !has_trailers => has_trailers = true,
+ _ => anyhow::bail!("expected request body"),
+ }
+ }
+ let (body, model) = router.decide(&body, &headers).await?;
+ if body.len() > MAX_BODY {
+ return Err(Error::new(413, "rewritten request body exceeds 2 MiB").into());
+ }
+ remove_headers.push("content-length".into());
+ Ok(Prepared {
+ body,
+ headers: ext::HeaderMutation {
+ set_headers: vec![set_header(MODEL_HEADER, &model)],
+ remove_headers,
+ },
+ has_trailers,
+ })
+ }
+}
+
+#[tonic::async_trait]
+impl ExternalProcessor for Server {
+ type ProcessStream =
+ Pin<Box<dyn Stream<Item = Result<ext::ProcessingResponse, Status>> + Send>>;
+
+ async fn process(
+ &self,
+ request: Request<Streaming<ext::ProcessingRequest>>,
+ ) -> Result<Response<Self::ProcessStream>, Status> {
+ let stream_permit = self.streams.clone().try_acquire_owned();
+ let request_permit = stream_permit
+ .as_ref()
+ .ok()
+ .and_then(|_| self.requests.clone().try_acquire_owned().ok());
+ let router = self.router.clone();
+ let deadline = Instant::now() + REQUEST_TIMEOUT;
+ let mut input = request.into_inner();
+ let output = async_stream::stream! {
+ // These permits belong to the output stream, including while it is backpressured.
+ // Dropping that stream also cancels input reads and the routing future.
+ let Ok(_stream_permit) = stream_permit else {
+ yield Ok(reject(Error::new(503, "processor stream limit reached")));
+ return;
+ };
+ let Some(request_permit) = request_permit else {
+ yield Ok(reject(Error::new(503, "preprocessor concurrency limit reached")));
+ return;
+ };
+ let result = timeout_at(deadline, Self::prepare(&router, &mut input)).await;
+ drop(request_permit);
+ let prepared = match result {
+ Ok(Ok(prepared)) => prepared,
+ Ok(Err(error)) => {
+ tracing::warn!(%error, "preprocessing rejected request");
+ yield Ok(reject(error.downcast::<Error>()
+ .unwrap_or_else(|e| Error::new(400, e.to_string()))));
+ return;
+ }
+ Err(_) => {
+ yield Ok(reject(Error::new(504, "preprocessing deadline exceeded")));
+ return;
+ }
+ };
+ let mut response_deadline = Instant::now() + RESPONSE_TIMEOUT;
+ // v1.0 selects the route when this header reply arrives, before reading the body.
+ yield Ok(reply(Output::RequestHeaders(ext::HeadersResponse {
+ response: Some(ext::CommonResponse {
+ header_mutation: Some(prepared.headers),
+ ..Default::default()
+ }),
+ })));
+ if prepared.has_trailers {
+ yield Ok(reply(Output::RequestTrailers(ext::TrailersResponse::default())));
+ }
+ for (i, chunk) in prepared.body.chunks(CHUNK_SIZE).enumerate() {
+ if Instant::now() >= response_deadline {
+ yield Err(Status::deadline_exceeded("request output timed out"));
+ return;
+ }
+ yield Ok(reply(Output::RequestBody(body_reply(
+ chunk.to_vec(), (i + 1) * CHUNK_SIZE >= prepared.body.len(),
+ ))));
+ }
+ response_deadline = Instant::now() + RESPONSE_TIMEOUT;
+ let mut has_headers = false;
+ let mut has_trailers = false;
+ loop {
+ let next = match timeout_at(response_deadline, input.message()).await {
+ Ok(Ok(Some(next))) => next,
+ Ok(Ok(None)) => {
+ yield Err(Status::failed_precondition("incomplete response stream"));
+ return;
+ }
+ Ok(Err(error)) => { yield Err(error); return; }
+ Err(_) => {
+ yield Err(Status::deadline_exceeded("response processing timed out"));
+ return;
+ }
+ };
+ response_deadline = Instant::now() + RESPONSE_TIMEOUT;
+ let (output, is_done) = match next.request {
+ Some(Input::ResponseHeaders(h)) if !has_headers => {
+ has_headers = true;
+ (Output::ResponseHeaders(ext::HeadersResponse {
+ response: Some(ext::CommonResponse::default()),
+ }), h.end_of_stream)
+ }
+ Some(Input::ResponseBody(b)) if has_headers
+ && (!has_trailers || (b.body.is_empty() && b.end_of_stream)) => {
+ let is_done = b.end_of_stream;
+ (Output::ResponseBody(body_reply(b.body, is_done)), is_done)
+ }
+ Some(Input::ResponseTrailers(_)) if has_headers && !has_trailers => {
+ has_trailers = true;
+ // ACK trailers, then consume agentgateway's synthetic body EOS.
+ (Output::ResponseTrailers(ext::TrailersResponse::default()), false)
+ }
+ _ => {
+ yield Err(Status::invalid_argument("unexpected response processing phase"));
+ return;
+ }
+ };
+ yield Ok(reply(output));
+ if is_done { return; }
+ }
+ };
+ Ok(Response::new(Box::pin(with_output_timeout(output))))
+ }
+}
+
+fn with_output_timeout(
+ output: impl Stream<Item = Result<ext::ProcessingResponse, Status>> + Send + 'static,
+) -> impl Stream<Item = Result<ext::ProcessingResponse, Status>> + Send {
+ let (tx, mut rx) = mpsc::channel(1);
+ let mut tasks = JoinSet::new();
+ // Poll independently of tonic so a stalled reader cannot retain admission permits.
+ tasks.spawn(async move {
+ let mut output = pin!(output);
+ while let Some(response) = output.next().await {
+ if timeout(RESPONSE_TIMEOUT, tx.send(response))
+ .await
+ .map_err(|_| Status::deadline_exceeded("processor output stalled"))?
+ .is_err()
+ {
+ break;
+ }
+ }
+ Ok::<_, Status>(())
+ });
+ async_stream::stream! {
+ // Owning the JoinSet also cancels the producer when this stream is dropped.
+ while let Some(response) = rx.recv().await {
+ yield response;
+ }
+ match tasks.join_next().await {
+ Some(Ok(Err(error))) => yield Err(error),
+ Some(Err(_)) => yield Err(Status::internal("processor task failed")),
+ _ => {}
+ }
+ }
+}
+
+fn reply(response: Output) -> ext::ProcessingResponse {
+ ext::ProcessingResponse {
+ response: Some(response),
+ ..Default::default()
+ }
+}
+
+fn body_reply(body: Vec<u8>, is_end_of_stream: bool) -> ext::BodyResponse {
+ ext::BodyResponse {
+ response: Some(ext::CommonResponse {
+ body_mutation: Some(ext::BodyMutation {
+ mutation: Some(ext::body_mutation::Mutation::StreamedResponse(
+ ext::StreamedBodyResponse {
+ body,
+ end_of_stream: is_end_of_stream,
+ },
+ )),
+ }),
+ ..Default::default()
+ }),
+ }
+}
+
+fn set_header(key: &str, value: &str) -> HeaderValueOption {
+ HeaderValueOption {
+ header: Some(HeaderValue {
+ key: key.into(),
+ value: String::new(),
+ raw_value: value.as_bytes().to_vec(),
+ }),
+ append_action: 2,
+ keep_empty_value: false,
+ ..Default::default()
+ }
+}
+
+fn reject(error: Error) -> ext::ProcessingResponse {
+ reply(Output::ImmediateResponse(ext::ImmediateResponse {
+ status: Some(crate::proto::envoy::r#type::v3::HttpStatus {
+ code: i32::from(error.status_code),
+ }),
+ headers: Some(ext::HeaderMutation {
+ set_headers: vec![set_header("content-type", "application/json")],
+ remove_headers: vec![],
+ }),
+ body: serde_json::to_vec(&serde_json::json!({"error": {
+ "message": error.message, "type": "switchyard_preproc_error"
+ }}))
+ .expect("JSON error object serializes"),
+ details: "switchyard_preproc".into(),
+ ..Default::default()
+ }))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream};
+
+ type Client =
+ ext::external_processor_client::ExternalProcessorClient<tonic::transport::Channel>;
+
+ async fn start(max_active_streams: usize) -> (Client, Arc<Semaphore>) {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let runner = switchyard_runner::Runner::from_toml(include_str!("../config/routes.toml"));
+ let server = Server::new(Arc::new(Router::new(runner.unwrap())), max_active_streams);
+ let requests = server.requests.clone();
+ tokio::spawn(async move {
+ tonic::transport::Server::builder()
+ .add_service(server.into_service())
+ .serve_with_incoming(TcpListenerStream::new(listener))
+ .await
+ .unwrap();
+ });
+ (
+ Client::new(
+ tonic::transport::Endpoint::from_shared(format!("http://{address}"))
+ .unwrap()
+ .connect()
+ .await
+ .unwrap(),
+ ),
+ requests,
+ )
+ }
+
+ fn input(request: Input) -> ext::ProcessingRequest {
+ ext::ProcessingRequest {
+ request: Some(request),
+ ..Default::default()
+ }
+ }
+
+ fn body(bytes: &[u8], is_end_of_stream: bool) -> ext::HttpBody {
+ ext::HttpBody {
+ body: bytes.to_vec(),
+ end_of_stream: is_end_of_stream,
+ }
+ }
+
+ fn headers() -> Input {
+ let headers = [
+ (":method", "POST"),
+ (":path", "/v1/chat/completions"),
+ ("content-type", "application/json"),
+ (MODEL_HEADER, "forged"),
+ ]
+ .into_iter()
+ .map(|(k, v)| set_header(k, v).header.unwrap())
+ .collect();
+ Input::RequestHeaders(ext::HttpHeaders {
+ headers: Some(crate::proto::envoy::config::core::v3::HeaderMap { headers }),
+ end_of_stream: false,
+ ..Default::default()
+ })
+ }
+
+ async fn send(tx: &mpsc::Sender<ext::ProcessingRequest>, request: Input) {
+ tx.send(input(request)).await.unwrap();
+ }
+
+ async fn next(rx: &mut Streaming<ext::ProcessingResponse>) -> Output {
+ rx.message().await.unwrap().unwrap().response.unwrap()
+ }
+
+ fn streamed(reply: ext::BodyResponse) -> ext::StreamedBodyResponse {
+ let Some(ext::body_mutation::Mutation::StreamedResponse(body)) =
+ reply.response.unwrap().body_mutation.unwrap().mutation
+ else {
+ panic!()
+ };
+ body
+ }
+
+ #[tokio::test]
+ async fn routes_upload_and_passes_response_through_trailers_to_eos() {
+ let (mut client, requests) = start(16).await;
+ let (tx, receive) = mpsc::channel(8);
+ send(&tx, headers()).await;
+ let receive = ReceiverStream::new(receive);
+ let mut rx = client.process(receive).await.unwrap().into_inner();
+ let json = br#"{"model":"auto","messages":[{"role":"user","content":"hello"}]}"#;
+ for chunk in json.chunks(40) {
+ send(&tx, Input::RequestBody(body(chunk, false))).await;
+ }
+ assert!(
+ tokio::time::timeout(Duration::from_millis(30), rx.message())
+ .await
+ .is_err()
+ );
+ send(&tx, Input::RequestTrailers(ext::HttpTrailers::default())).await;
+ send(&tx, Input::RequestBody(body(b"", true))).await;
+ let Output::RequestHeaders(h) = next(&mut rx).await else {
+ panic!()
+ };
+ let h = h.response.unwrap().header_mutation.unwrap();
+ let selected = h.set_headers[0].header.as_ref().unwrap();
+ assert_eq!(selected.raw_value, b"Qwen/Qwen3-0.6B");
+ assert!(h.remove_headers.contains(&"content-length".to_owned()));
+ assert!(matches!(next(&mut rx).await, Output::RequestTrailers(_)));
+ let Output::RequestBody(b) = next(&mut rx).await else {
+ panic!()
+ };
+ let b = streamed(b);
+ assert!(b.end_of_stream);
+ let value: serde_json::Value = serde_json::from_slice(&b.body).unwrap();
+ assert_eq!(value["model"], "Qwen/Qwen3-0.6B");
+ assert_eq!(requests.available_permits(), 8);
+ send(&tx, Input::ResponseHeaders(ext::HttpHeaders::default())).await;
+ assert!(matches!(next(&mut rx).await, Output::ResponseHeaders(_)));
+ for (bytes, is_done) in [
+ (b"data: hello\n\n".as_slice(), false),
+ (b"".as_slice(), true),
+ ] {
+ send(&tx, Input::ResponseBody(body(bytes, is_done))).await;
+ let Output::ResponseBody(b) = next(&mut rx).await else {
+ panic!()
+ };
+ let b = streamed(b);
+ assert_eq!((b.body.as_slice(), b.end_of_stream), (bytes, is_done));
+ if !is_done {
+ send(&tx, Input::ResponseTrailers(ext::HttpTrailers::default())).await;
+ assert!(matches!(next(&mut rx).await, Output::ResponseTrailers(_)));
+ }
+ }
+ assert!(rx.message().await.unwrap().is_none());
+ }
+
+ #[tokio::test]
+ async fn response_timeout_tracks_inactivity() {
+ async fn elapse(duration: Duration) {
+ // Resume before socket I/O so idle network polls cannot auto-advance time.
+ tokio::time::pause();
+ tokio::time::advance(duration).await;
+ tokio::time::resume();
+ }
+
+ let (mut client, _) = start(1).await;
+ let (tx, receive) = mpsc::channel(8);
+ send(&tx, headers()).await;
+ let mut rx = client
+ .process(ReceiverStream::new(receive))
+ .await
+ .unwrap()
+ .into_inner();
+ send(
+ &tx,
+ Input::RequestBody(body(
+ br#"{"model":"auto","messages":[{"role":"user","content":"hello"}]}"#,
+ true,
+ )),
+ )
+ .await;
+ assert!(matches!(next(&mut rx).await, Output::RequestHeaders(_)));
+ assert!(matches!(next(&mut rx).await, Output::RequestBody(_)));
+ send(&tx, Input::ResponseHeaders(ext::HttpHeaders::default())).await;
+ assert!(matches!(next(&mut rx).await, Output::ResponseHeaders(_)));
+
+ let mut rejected = client
+ .process(tokio_stream::iter([input(headers())]))
+ .await
+ .unwrap()
+ .into_inner();
+ let Output::ImmediateResponse(error) = next(&mut rejected).await else {
+ panic!()
+ };
+ assert_eq!(error.status.unwrap().code, 503);
+
+ for _ in 0..3 {
+ elapse(RESPONSE_TIMEOUT / 2).await;
+ send(&tx, Input::ResponseBody(body(b"data: hello\n\n", false))).await;
+ let Output::ResponseBody(b) = next(&mut rx).await else {
+ panic!()
+ };
+ assert_eq!(streamed(b).body, b"data: hello\n\n");
+ }
+ elapse(RESPONSE_TIMEOUT + Duration::from_secs(1)).await;
+ assert_eq!(
+ rx.message().await.unwrap_err().code(),
+ tonic::Code::DeadlineExceeded
+ );
+ }
+
+ #[tokio::test]
+ async fn rejects_cumulative_body_limit_before_routing_headers() {
+ let (mut client, _) = start(16).await;
+ let messages = tokio_stream::iter([
+ input(headers()),
+ input(Input::RequestBody(body(&vec![b' '; MAX_BODY], false))),
+ input(Input::RequestBody(body(b" ", true))),
+ ]);
+ let mut rx = client.process(messages).await.unwrap().into_inner();
+ let Output::ImmediateResponse(error) = next(&mut rx).await else {
+ panic!()
+ };
+ assert_eq!(error.status.unwrap().code, 413);
+ assert!(rx.message().await.unwrap().is_none());
+ }
+}