Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(relay): migrate dynamic plugin to `switchyard-runner` (#528)

Bryan Bednarski074b2bec3f16
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e9b7fa6dd..76884e283 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,16 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
### Added
+- **NeMo Relay native plugin** — a dynamically loaded integration that loads
+ Switchyard's standard TOML deployment and executes its `switchyard-runner`-
+ supported configured routes in process. Managed calls require NeMo Relay
+ `>=0.8.1,<0.9.0`; unknown models use Relay's continuation unchanged.
+
+- **NeMo Relay routing marks** — routing-model usage, measured routing
+ overhead, and selected-model decisions are emitted as ATOF marks. The final
+ serving call remains represented only by Relay's outer LLM lifecycle event to
+ prevent double-counting.
+
- **Advisor-gate routing** — new `advisor` route type pairing the serving
executor with a stronger judge-only advisor that reviews terminal turns:
APPROVE releases the buffered turn, REDO discards it and feeds the advisor's
diff --git a/Cargo.lock b/Cargo.lock
index 89b712d9d..ab1f6c505 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -274,6 +274,9 @@ name = "bitflags"
version = "2.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
+dependencies = [
+ "serde_core",
+]
[[package]]
name = "borrow-or-share"
@@ -334,6 +337,16 @@ dependencies = [
"rand_core 0.10.1",
]
+[[package]]
+name = "chrono"
+version = "0.4.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
+dependencies = [
+ "num-traits",
+ "serde",
+]
+
[[package]]
name = "clap"
version = "4.6.2"
@@ -1226,6 +1239,34 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "nemo-relay-plugin"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ab4eaa56f021794a8b858c1ec6783679ad6798653e00a179166182bc4868bec"
+dependencies = [
+ "futures",
+ "nemo-relay-types",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tokio-util",
+]
+
+[[package]]
+name = "nemo-relay-types"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bd5df5a9286dd167f668939decc7c719c6b7e4253a2f758b5fd3b8af22cc3b0"
+dependencies = [
+ "bitflags",
+ "chrono",
+ "serde",
+ "serde_json",
+ "typed-builder",
+ "uuid",
+]
+
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@@ -2325,6 +2366,25 @@ dependencies = [
"wiremock",
]
+[[package]]
+name = "switchyard-nemo-relay-plugin"
+version = "0.2.0"
+dependencies = [
+ "futures-util",
+ "http",
+ "nemo-relay-plugin",
+ "serde",
+ "serde_json",
+ "switchyard-llm-client",
+ "switchyard-protocol",
+ "switchyard-runner",
+ "switchyard-translation",
+ "tempfile",
+ "tokio",
+ "toml",
+ "wiremock",
+]
+
[[package]]
name = "switchyard-protocol"
version = "0.2.0"
@@ -2644,6 +2704,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-sink",
+ "futures-util",
"pin-project-lite",
"tokio",
]
@@ -2817,6 +2878,26 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+[[package]]
+name = "typed-builder"
+version = "0.23.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda"
+dependencies = [
+ "typed-builder-macro",
+]
+
+[[package]]
+name = "typed-builder-macro"
+version = "0.23.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
[[package]]
name = "unicode-general-category"
version = "1.1.0"
@@ -2859,6 +2940,18 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+[[package]]
+name = "uuid"
+version = "1.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"
+dependencies = [
+ "getrandom 0.3.4",
+ "js-sys",
+ "serde",
+ "wasm-bindgen",
+]
+
[[package]]
name = "uuid-simd"
version = "0.8.0"
diff --git a/Cargo.toml b/Cargo.toml
index 6cacbd29e..7a8fc60ef 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ members = [
"crates/prefill-router",
"crates/switchyard-py",
"crates/protocol",
+ "crates/switchyard-nemo-relay-plugin",
"crates/switchyard-runner",
"crates/switchyard-server",
"crates/switchyard-skill-distillation",
@@ -34,6 +35,7 @@ http = "1"
httpdate = "1"
jsonschema = { version = "0.49.4", default-features = false }
jsonptr = { version = "0.8.1", default-features = false, features = ["std", "json", "resolve"] }
+nemo-relay-plugin = ">=0.8.1, <0.9.0"
parking_lot = "0.12"
rand = "0.10"
regex = "1"
diff --git a/README.md b/README.md
index 265bb56b0..ded0df8f0 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,7 @@ algorithm you write yourself.
- **Protocol Translation**: convert between OpenAI Chat, Anthropic Messages, and OpenAI Responses formats
- **Multi-Backend Routing**: random routing, LLM-as-classifier routing, signal-driven stage-router, or your own algorithm
- **Operational Metrics**: Prometheus metrics cover requests, errors, latency, tokens, and routing overhead
+- **NeMo Relay Plugin**: run Switchyard-configured routes in NeMo Relay—including any routing algorithm supported by `switchyard-runner`—while Switchyard owns provider HTTP dispatch
## Maturity
@@ -129,6 +130,7 @@ configured LLM client selects one upstream format.
- **[`switchyard-libsy`](crates/libsy/README.md)**: embed routing algorithms in a Rust application
- **[`switchyard-protocol`](crates/protocol/README.md)**: provider-neutral request, response, and streaming types
- **[`switchyard-translation`](crates/switchyard-translation/README.md)**: request, response, and stream translation
+- **[`switchyard-nemo-relay-plugin`](crates/switchyard-nemo-relay-plugin/README.md)**: install Switchyard as a native NeMo Relay plugin
## Community
diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml
new file mode 100644
index 000000000..ac9099f7f
--- /dev/null
+++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml
@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "switchyard-nemo-relay-plugin"
+version.workspace = true
+description = "Switchyard-owned HTTP routing plugin for NeMo Relay"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+futures-util.workspace = true
+http.workspace = true
+nemo-relay-plugin.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+switchyard-llm-client.workspace = true
+switchyard-protocol.workspace = true
+switchyard-runner.workspace = true
+switchyard-translation.workspace = true
+tokio.workspace = true
+toml = "1.1"
+
+[dev-dependencies]
+tempfile = "3"
+wiremock = "0.6"
diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md
new file mode 100644
index 000000000..d1359096a
--- /dev/null
+++ b/crates/switchyard-nemo-relay-plugin/README.md
@@ -0,0 +1,118 @@
+# Switchyard NeMo Relay Plugin
+
+`switchyard-nemo-relay-plugin` is a native NeMo Relay dynamic plugin. It loads
+a standard Switchyard TOML deployment from a file or Relay's nested plugin
+configuration and executes its configured routes through `switchyard-runner`.
+
+The plugin does not define a second routing or target configuration language.
+`switchyard-server` and Relay therefore use the same targets, client pooling,
+algorithm construction, retry policy, and route validation.
+
+## Install
+
+Build the platform bundle with the package script, then configure Relay to load
+the generated `relay-plugin.toml` manifest. The plugin requires NeMo Relay
+`>=0.8.1,<0.9.0`.
+
+## Configure Relay
+
+Use exactly one Switchyard deployment source. To share an existing deployment
+file with `switchyard-server`, configure its path:
+
+```toml
+[[plugins.dynamic]]
+manifest = "./plugins/switchyard/relay-plugin.toml"
+
+[plugins.dynamic.config]
+priority = 0
+switchyard_config_path = "/etc/switchyard/routes.toml"
+```
+
+`switchyard_config_path` is a Switchyard version-1 TOML deployment, accepted by both
+`switchyard-server` and `switchyard-runner`. See the
+[server configuration guide](../switchyard-server/CONFIGURATION.md) for the
+deployment schema and routing algorithms.
+
+To keep the deployment in the Relay configuration, nest the same version-1
+Switchyard configuration under `switchyard_config`:
+
+```toml
+[[plugins.dynamic]]
+manifest = "./plugins/switchyard/relay-plugin.toml"
+
+[plugins.dynamic.config]
+priority = 0
+
+[plugins.dynamic.config.switchyard_config]
+schema_version = 1
+
+[plugins.dynamic.config.switchyard_config.llm_clients.primary]
+format = "openai_chat"
+base_url = "https://example.test/v1"
+
+[plugins.dynamic.config.switchyard_config.targets.default]
+id = "example/model"
+llm_client = "primary"
+
+[plugins.dynamic.config.switchyard_config.routes.default]
+id = "switchyard/default"
+type = "passthrough"
+target = "default"
+```
+
+## Request handling
+
+For OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages calls,
+the plugin decodes the Relay request and checks the requested model against the
+deployment's route IDs.
+
+- A configured route is executed by `switchyard-runner`.
+- An unknown model calls Relay's continuation unchanged.
+- The returned provider response is encoded back into the caller's wire format.
+- Streaming responses are returned as unpolled translated streams; Relay owns
+ cancellation and the outer serving-call lifecycle.
+
+Each route's target client must use the caller's wire format: `openai_chat`,
+`openai_responses`, or `anthropic_messages`. The runner selects the upstream
+backend from that format rather than translating a route to a different
+provider API. When one upstream model must serve multiple caller formats,
+declare a target and route for each corresponding client format.
+
+The plugin emits a routing request mark, routing-model call marks, measured
+routing-overhead marks, and a selected-model decision mark. Token usage is
+emitted as Switchyard metrics for both routing-model and answer-model calls;
+Relay retains ownership of the outer LLM lifecycle.
+
+## Observability
+
+When Relay is configured with OTLP logs and metrics exporters, the plugin emits
+typed telemetry through Relay's native plugin runtime:
+
+- Routing request, decision, and overhead marks are Info logs.
+- Per-routing-model call marks are Debug logs, including their outcome and
+ latency, but not token usage.
+- Terminal routing and response-finalization failures are Error logs. Their
+ payload contains only the safe Switchyard failure summary; it excludes
+ provider response bodies and free-form provider messages.
+- Metrics use bounded attributes only: algorithm for
+ `switchyard.routing.requests`; outcome for `switchyard.routing.llm_calls`
+ and `switchyard.routing.llm_call.duration`;
+ and safe failure kind, category, phase, and optional upstream HTTP status for
+ `switchyard.routing.failures`. `switchyard.routing.overhead` records total
+ routing latency, including routing-model calls; durations use milliseconds.
+- `switchyard.routing.llm_tokens` records normalized token usage with
+ `call_role` (`routing` or `answer`), configured `target_model`, and
+ `token_type` attributes. A provider may omit usage for streaming responses;
+ the plugin does not synthesize zero-value measurements.
+
+The plugin does not attach sessions, requests, or provider messages as metric
+attributes. `target_model` comes from the configured Switchyard target set,
+rather than arbitrary caller input, keeping the metric cardinality bounded by
+the deployment.
+
+## Failure policy
+
+`switchyard-llm-client` owns provider retry and route-candidate fallback
+behavior. The plugin does not maintain a separate trusted-default target or
+rerun routing after an execution failure. Failures outside the shared runner,
+including response translation failures, are returned to Relay.
diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json
new file mode 100644
index 000000000..61f178a12
--- /dev/null
+++ b/crates/switchyard-nemo-relay-plugin/config.schema.json
@@ -0,0 +1,40 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "title": "Switchyard NeMo Relay Plugin",
+ "description": "Runs a Switchyard deployment through its shared in-process runner.",
+ "type": "object",
+ "additionalProperties": false,
+ "oneOf": [
+ { "required": ["switchyard_config_path"] },
+ { "required": ["switchyard_config"] }
+ ],
+ "properties": {
+ "priority": {
+ "type": "integer",
+ "default": 0,
+ "description": "NeMo Relay execution-intercept priority."
+ },
+ "executor": {
+ "type": "object",
+ "additionalProperties": false,
+ "description": "Optional NeMo Relay SDK executor override.",
+ "properties": {
+ "worker_threads": {
+ "type": "integer",
+ "minimum": 1,
+ "description": "Number of worker threads for the plugin's SDK-owned executor."
+ }
+ }
+ },
+ "switchyard_config_path": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Path to a Switchyard version-1 TOML deployment shared with switchyard-server."
+ },
+ "switchyard_config": {
+ "type": "object",
+ "minProperties": 1,
+ "description": "Inline Switchyard version-1 deployment using the same schema as switchyard-server."
+ }
+ }
+}
diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml
new file mode 100644
index 000000000..e2f23aa17
--- /dev/null
+++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml
@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+manifest_version = 1
+
+[plugin]
+id = "nvidia.switchyard"
+kind = "rust_dynamic"
+
+[compat]
+relay = ">=0.8.1,<0.9.0"
+native_api = "1"
+
+[defaults]
+enabled = false
+
+[capabilities]
+items = ["plugin_native", "config_schema"]
+
+[config_schema]
+path = "config.schema.json"
+
+[source]
+artifact = "<platform-library-file>"
+
+[integrity]
+sha256 = "sha256:<artifact-sha256>"
+
+[load]
+library = "<platform-library-file>"
+symbol = "nemo_relay_register_plugin"
diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py
new file mode 100644
index 000000000..3b79af769
--- /dev/null
+++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py
@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Materialize the minimal Relay plugin bundle from a compiled cdylib."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import shutil
+import tarfile
+import zipfile
+from pathlib import Path
+
+CRATE_ROOT = Path(__file__).resolve().parents[1]
+REPOSITORY_ROOT = CRATE_ROOT.parents[1]
+PACKAGE_NAME = "switchyard-nemo-relay-plugin"
+
+
+def digest(path: Path) -> str:
+ """Return the lowercase SHA-256 digest for a file."""
+ value = hashlib.sha256()
+ with path.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ value.update(chunk)
+ return value.hexdigest()
+
+
+def archive_bundle(bundle: Path, archive: Path) -> None:
+ """Archive a materialized bundle under the stable package directory name."""
+ if archive.parent.is_relative_to(bundle):
+ raise ValueError(f"bundle archive must be outside output directory: {archive}")
+ archive.parent.mkdir(parents=True, exist_ok=True)
+ if archive.exists():
+ raise ValueError(f"bundle archive already exists: {archive}")
+
+ if archive.name.endswith(".tar.gz"):
+ with tarfile.open(archive, "w:gz") as stream:
+ stream.add(bundle, arcname=PACKAGE_NAME)
+ return
+
+ if archive.suffix == ".zip":
+ with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as stream:
+ for path in sorted(bundle.rglob("*")):
+ if path.is_file():
+ stream.write(path, Path(PACKAGE_NAME) / path.relative_to(bundle))
+ return
+
+ raise ValueError("bundle archive must end in .tar.gz or .zip")
+
+
+def main() -> None:
+ """Materialize a Relay-loadable plugin bundle in an empty directory."""
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--library", required=True, type=Path)
+ parser.add_argument("--output", required=True, type=Path)
+ parser.add_argument("--archive", type=Path)
+ args = parser.parse_args()
+
+ library = args.library.resolve()
+ if not library.is_file():
+ parser.error(f"compiled plugin library does not exist: {library}")
+
+ manifest = (CRATE_ROOT / "relay-plugin.toml").read_text(encoding="utf-8")
+ placeholders = ("<platform-library-file>", "<artifact-sha256>")
+ missing = [placeholder for placeholder in placeholders if placeholder not in manifest]
+ if missing:
+ parser.error(f"plugin manifest is missing placeholders: {', '.join(missing)}")
+
+ output = args.output.resolve()
+ if output.exists() and not output.is_dir():
+ parser.error(f"bundle output exists and is not a directory: {output}")
+ if output.is_dir() and any(output.iterdir()):
+ parser.error(f"bundle output directory must be empty: {output}")
+
+ archive = args.archive.resolve() if args.archive is not None else None
+ if archive is not None and archive.parent.is_relative_to(output):
+ parser.error(f"bundle archive must be outside output directory: {archive}")
+ output.mkdir(parents=True, exist_ok=True)
+
+ artifact = output / library.name
+ shutil.copy2(library, artifact)
+ shutil.copy2(CRATE_ROOT / "config.schema.json", output / "config.schema.json")
+ for filename in ("LICENSE", "NOTICE"):
+ shutil.copy2(REPOSITORY_ROOT / filename, output / filename)
+
+ artifact_digest = digest(artifact)
+ manifest = manifest.replace("<platform-library-file>", artifact.name)
+ manifest = manifest.replace("<artifact-sha256>", artifact_digest)
+ (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8")
+
+ if archive is None:
+ print(output)
+ return
+
+ try:
+ archive_bundle(output, archive)
+ except ValueError as error:
+ parser.error(str(error))
+ print(archive)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs
new file mode 100644
index 000000000..2df6386b3
--- /dev/null
+++ b/crates/switchyard-nemo-relay-plugin/src/config.rs
@@ -0,0 +1,152 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::path::PathBuf;
+
+use serde::Deserialize;
+use serde_json::{Map, Value};
+use switchyard_protocol::WireFormat;
+use switchyard_runner::Runner;
+
+pub(crate) fn protocol_from_call(name: &str) -> Option<WireFormat> {
+ match name {
+ "openai.chat_completions" => Some(WireFormat::OpenAiChat),
+ "openai.responses" => Some(WireFormat::OpenAiResponses),
+ "anthropic.messages" => Some(WireFormat::AnthropicMessages),
+ _ => None,
+ }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct SwitchyardConfig {
+ #[serde(default)]
+ pub(crate) priority: i32,
+ #[serde(default)]
+ pub(crate) switchyard_config_path: Option<PathBuf>,
+ #[serde(default)]
+ pub(crate) switchyard_config: Option<Map<String, Value>>,
+}
+
+impl SwitchyardConfig {
+ pub(crate) fn load_runner(&self) -> Result<Runner, String> {
+ match (&self.switchyard_config_path, &self.switchyard_config) {
+ (Some(path), None) => Runner::load(path).map_err(|error| error.to_string()),
+ (None, Some(config)) => toml::to_string(config)
+ .map_err(|error| format!("failed to serialize Switchyard configuration: {error}"))
+ .and_then(|source| Runner::from_toml(&source).map_err(|error| error.to_string())),
+ (Some(_), Some(_)) => Err(
+ "configure exactly one of switchyard_config_path or switchyard_config".to_string(),
+ ),
+ (None, None) => {
+ Err("configure one of switchyard_config_path or switchyard_config".to_string())
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::io::Write;
+
+ use serde_json::json;
+ use tempfile::NamedTempFile;
+
+ use super::*;
+
+ fn inline_deployment() -> Map<String, Value> {
+ json!({
+ "schema_version": 1,
+ "llm_clients": {
+ "primary": {
+ "format": "openai_chat",
+ "base_url": "https://example.test/v1"
+ }
+ },
+ "targets": {
+ "default": {
+ "id": "example/model",
+ "llm_client": "primary"
+ }
+ },
+ "routes": {
+ "default": {
+ "id": "switchyard/default",
+ "type": "passthrough",
+ "target": "default"
+ }
+ }
+ })
+ .as_object()
+ .unwrap()
+ .clone()
+ }
+
+ #[test]
+ fn maps_supported_relay_call_names_to_wire_formats() {
+ let cases = [
+ ("openai.chat_completions", Some(WireFormat::OpenAiChat)),
+ ("openai.responses", Some(WireFormat::OpenAiResponses)),
+ ("anthropic.messages", Some(WireFormat::AnthropicMessages)),
+ ("unsupported.call", None),
+ ];
+
+ for (name, expected) in cases {
+ assert_eq!(protocol_from_call(name), expected, "{name}");
+ }
+ }
+
+ #[test]
+ fn builds_runner_from_inline_deployment() {
+ let config = SwitchyardConfig {
+ priority: 0,
+ switchyard_config_path: None,
+ switchyard_config: Some(inline_deployment()),
+ };
+
+ let runner = config.load_runner().unwrap();
+ assert!(runner.route("switchyard/default").is_some());
+ }
+
+ #[test]
+ fn builds_runner_from_deployment_path() {
+ let mut deployment = NamedTempFile::new().unwrap();
+ write!(
+ deployment,
+ "{}",
+ toml::to_string(&inline_deployment()).unwrap()
+ )
+ .unwrap();
+ let config = SwitchyardConfig {
+ priority: 0,
+ switchyard_config_path: Some(deployment.path().to_path_buf()),
+ switchyard_config: None,
+ };
+
+ let runner = config.load_runner().unwrap();
+ assert!(runner.route("switchyard/default").is_some());
+ }
+
+ #[test]
+ fn requires_exactly_one_configuration_source() {
+ let neither = SwitchyardConfig {
+ priority: 0,
+ switchyard_config_path: None,
+ switchyard_config: None,
+ };
+ assert!(matches!(
+ neither.load_runner(),
+ Err(error) if error.contains("configure one of switchyard_config_path or switchyard_config")
+ ));
+
+ let both = SwitchyardConfig {
+ priority: 0,
+ switchyard_config_path: Some(PathBuf::from("routes.toml")),
+ switchyard_config: Some(inline_deployment()),
+ };
+ assert!(matches!(
+ both.load_runner(),
+ Err(error) if error.contains("configure exactly one of switchyard_config_path or switchyard_config")
+ ));
+ }
+}
diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs
new file mode 100644
index 000000000..853a2e81b
--- /dev/null
+++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs
@@ -0,0 +1,173 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+mod config;
+mod runtime;
+mod translation;
+
+use std::sync::Arc;
+
+use nemo_relay_plugin::{
+ ConfigDiagnostic, DiagnosticLevel, Json, NativePlugin, PluginContext, PluginRuntime,
+};
+use serde_json::Map;
+
+use crate::config::{SwitchyardConfig, protocol_from_call};
+use crate::runtime::{SwitchyardRuntime, emit_event, emit_events};
+
+#[derive(Default)]
+struct SwitchyardPlugin;
+
+impl NativePlugin for SwitchyardPlugin {
+ fn plugin_kind(&self) -> &str {
+ "nvidia.switchyard"
+ }
+
+ fn allows_multiple_components(&self) -> bool {
+ false
+ }
+
+ fn validate(&self, plugin_config: &Map<String, Json>) -> Vec<ConfigDiagnostic> {
+ // Allow `SwitchyardRuntime::new` to run for accurate validation diagnostics.
+ match parse_config(plugin_config).and_then(SwitchyardRuntime::new) {
+ Ok(_) => Vec::new(),
+ Err(message) => vec![ConfigDiagnostic {
+ level: DiagnosticLevel::Error,
+ code: "switchyard.invalid_config".into(),
+ component: Some("nvidia.switchyard".into()),
+ field: Some("config".into()),
+ message,
+ }],
+ }
+ }
+
+ fn register(
+ &mut self,
+ plugin_config: &Map<String, Json>,
+ ctx: &mut PluginContext<'_>,
+ ) -> nemo_relay_plugin::Result<()> {
+ let config = parse_config(plugin_config)?;
+ let priority = config.priority;
+ let runtime = Arc::new(SwitchyardRuntime::new(config)?);
+ let plugin_runtime = ctx.runtime();
+ register_buffered(ctx, priority, Arc::clone(&runtime), plugin_runtime.clone())?;
+ register_stream(ctx, priority, runtime, plugin_runtime)?;
+ Ok(())
+ }
+}
+
+fn register_buffered(
+ ctx: &mut PluginContext<'_>,
+ priority: i32,
+ runtime: Arc<SwitchyardRuntime>,
+ plugin_runtime: PluginRuntime,
+) -> Result<(), String> {
+ ctx.register_llm_execution_intercept(
+ "switchyard.runner.buffered",
+ priority,
+ move |name, request, next| {
+ let runtime = Arc::clone(&runtime);
+ let plugin_runtime = plugin_runtime.clone();
+ async move {
+ let Some(inbound) = protocol_from_call(&name) else {
+ return next.call(request).await;
+ };
+ if !request_model(&request).is_some_and(|model| runtime.manages_model(model)) {
+ return next.call(request).await;
+ }
+ let decoded = runtime.decode_request(inbound, request, false)?;
+ let execution = runtime.execute_buffered(inbound, decoded).await;
+ emit_events(&plugin_runtime, execution.events);
+ execution.result
+ }
+ },
+ )
+}
+
+fn register_stream(
+ ctx: &mut PluginContext<'_>,
+ priority: i32,
+ runtime: Arc<SwitchyardRuntime>,
+ plugin_runtime: PluginRuntime,
+) -> Result<(), String> {
+ ctx.register_llm_stream_execution_intercept(
+ "switchyard.runner.streaming",
+ priority,
+ move |name, request, next| {
+ let runtime = Arc::clone(&runtime);
+ let plugin_runtime = plugin_runtime.clone();
+ async move {
+ let Some(inbound) = protocol_from_call(&name) else {
+ return next.call(request).await;
+ };
+ if !request_model(&request).is_some_and(|model| runtime.manages_model(model)) {
+ return next.call(request).await;
+ }
+ let decoded = runtime.decode_request(inbound, request, true)?;
+ let stream_plugin_runtime = plugin_runtime.clone();
+ let execution = runtime
+ .execute_stream(
+ inbound,
+ decoded,
+ Arc::new(move |event| emit_event(&stream_plugin_runtime, event)),
+ )
+ .await;
+ emit_events(&plugin_runtime, execution.events);
+ execution.result
+ }
+ },
+ )
+}
+
+fn parse_config(plugin_config: &Map<String, Json>) -> Result<SwitchyardConfig, String> {
+ let mut config = plugin_config.clone();
+ config.remove("executor");
+ serde_json::from_value(Json::Object(config))
+ .map_err(|error| format!("invalid Switchyard configuration: {error}"))
+}
+
+fn request_model(request: &nemo_relay_plugin::LlmRequest) -> Option<&str> {
+ request.content.get("model").and_then(Json::as_str)
+}
+
+nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default);
+
+#[cfg(test)]
+mod tests {
+ use serde_json::json;
+
+ use super::*;
+
+ #[test]
+ fn plugin_configuration_requires_a_switchyard_source() {
+ let config = json!({"priority": 0});
+ assert!(matches!(
+ parse_config(config.as_object().unwrap()).and_then(SwitchyardRuntime::new),
+ Err(error) if error.contains("switchyard_config_path or switchyard_config")
+ ));
+ }
+
+ #[test]
+ fn plugin_configuration_ignores_the_sdk_executor_override() {
+ let config = json!({
+ "executor": {"worker_threads": 4},
+ "switchyard_config_path": "routes.toml"
+ });
+ assert!(parse_config(config.as_object().unwrap()).is_ok());
+ }
+
+ #[test]
+ fn request_model_reads_only_a_top_level_string() {
+ let request = nemo_relay_plugin::LlmRequest {
+ headers: Map::new(),
+ content: json!({"model": "switchyard/default", "messages": []}),
+ };
+ assert_eq!(request_model(&request), Some("switchyard/default"));
+
+ let request = nemo_relay_plugin::LlmRequest {
+ headers: Map::new(),
+ content: json!({"model": {"name": "switchyard/default"}}),
+ };
+ assert_eq!(request_model(&request), None);
+ }
+}
diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs
new file mode 100644
index 000000000..7b5a12fb9
--- /dev/null
+++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs
@@ -0,0 +1,1412 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+
+use futures_util::{Stream, StreamExt};
+use http::StatusCode;
+use nemo_relay_plugin::{
+ Json, LlmRequest as RelayRequest, LogSeverity, MetricKind, MetricMeasurement, MetricValueType,
+ PluginRuntime,
+};
+use serde_json::{Map, json};
+use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver};
+use switchyard_protocol::{
+ LlmClientError, LlmResponse, LlmResponseChunk, LlmStreamError, Metadata, ProviderExtensions,
+ Request, Response, Usage, WireFormat,
+};
+use switchyard_runner::{Route, RouteErrorSummary, Runner, stream_error_summary};
+use switchyard_translation::{TranslationEngine, encode_stream_with_extensions};
+
+use crate::config::SwitchyardConfig;
+use crate::translation;
+
+#[derive(Debug)]
+pub(crate) struct RoutingMark {
+ pub(crate) name: String,
+ pub(crate) data: Json,
+ pub(crate) metadata: Json,
+ pub(crate) severity: Option<LogSeverity>,
+}
+
+#[derive(Debug)]
+pub(crate) struct RoutingMetric {
+ pub(crate) name: String,
+ pub(crate) measurements: Vec<MetricMeasurement>,
+ pub(crate) metadata: Json,
+}
+
+#[derive(Debug)]
+pub(crate) enum RoutingEvent {
+ Mark(RoutingMark),
+ Metric(RoutingMetric),
+}
+
+struct MetricDescriptor<'a> {
+ name: &'a str,
+ kind: MetricKind,
+ value_type: MetricValueType,
+ unit: Option<&'a str>,
+ description: &'a str,
+}
+
+pub(crate) type ReturnedEventStream = Pin<Box<dyn Stream<Item = Result<Json, String>> + Send>>;
+pub(crate) type RoutingEventEmitter = Arc<dyn Fn(RoutingEvent) + Send + Sync>;
+
+pub(crate) struct Execution<T> {
+ pub(crate) result: Result<T, String>,
+ pub(crate) events: Vec<RoutingEvent>,
+}
+
+pub(crate) struct SwitchyardRuntime {
+ runner: Runner,
+ translation: TranslationEngine,
+}
+
+impl SwitchyardRuntime {
+ pub(crate) fn new(config: SwitchyardConfig) -> Result<Self, String> {
+ Ok(Self {
+ runner: config.load_runner()?,
+ translation: TranslationEngine::default(),
+ })
+ }
+
+ pub(crate) fn manages_model(&self, model: &str) -> bool {
+ self.runner.route(model).is_some()
+ }
+
+ pub(crate) fn decode_request(
+ &self,
+ inbound: WireFormat,
+ request: RelayRequest,
+ streaming: bool,
+ ) -> Result<Request, String> {
+ let mut llm_request = translation::decode_request(&self.translation, inbound, &request)?;
+ llm_request.stream = streaming;
+ let headers = string_headers(&request.headers);
+ let mut metadata = Metadata::from_headers(&headers);
+ let relay_gateway_placeholder = !headers.contains_key("x-switchyard-session-id")
+ && headers
+ .get("x-nemo-relay-source")
+ .and_then(|value| value.to_str().ok())
+ == Some("gateway")
+ && metadata.session_id.as_deref() == Some("gateway-gateway");
+ if relay_gateway_placeholder {
+ metadata.session_id = None;
+ }
+ metadata.http_headers = Some(headers);
+ Ok(Request {
+ llm_request,
+ raw_request: Some(request.content),
+ metadata: Some(metadata),
+ })
+ }
+
+ pub(crate) async fn execute_buffered(
+ &self,
+ inbound: WireFormat,
+ request: Request,
+ ) -> Execution<Json> {
+ let request_extensions = request.llm_request.extensions.clone();
+ let Execution { result, mut events } = self.execute(inbound, request).await;
+ let (result, finalization_failed) = match result {
+ Ok(response) => {
+ let result = finalize_buffered_response(
+ &self.translation,
+ inbound,
+ response,
+ &request_extensions,
+ );
+ let failed = result.is_err();
+ (result, failed)
+ }
+ Err(error) => (Err(error), false),
+ };
+ if finalization_failed {
+ self.error_mark(&mut events, "response_finalization", None);
+ }
+ Execution { result, events }
+ }
+
+ pub(crate) async fn execute_stream(
+ &self,
+ inbound: WireFormat,
+ request: Request,
+ emit_event: RoutingEventEmitter,
+ ) -> Execution<ReturnedEventStream> {
+ let request_extensions = request.llm_request.extensions.clone();
+ let Execution { result, mut events } = self.execute(inbound, request).await;
+ let (result, finalization_failed) = match result {
+ Ok(response) => {
+ let metadata = events
+ .iter()
+ .find_map(|event| match event {
+ RoutingEvent::Mark(mark) => Some(mark.metadata.clone()),
+ RoutingEvent::Metric(_) => None,
+ })
+ .unwrap_or_else(|| Json::Object(Map::new()));
+ let result =
+ returned_events(response, inbound, &request_extensions, metadata, emit_event);
+ let failed = result.is_err();
+ (result, failed)
+ }
+ Err(error) => (Err(error), false),
+ };
+ if finalization_failed {
+ self.error_mark(&mut events, "response_finalization", None);
+ }
+ Execution { result, events }
+ }
+
+ async fn execute(&self, inbound: WireFormat, request: Request) -> Execution<Response> {
+ let Some(route) = self.route(&request) else {
+ return Execution {
+ result: Err("Switchyard has no route for this request model".into()),
+ events: Vec::new(),
+ };
+ };
+ let metadata = identity_metadata(request.metadata.as_ref());
+ let mut events = vec![RoutingEvent::Mark(RoutingMark {
+ name: "switchyard.routing.requested".into(),
+ data: json!({"algorithm": route.algorithm_name()}),
+ metadata: metadata.clone(),
+ severity: Some(LogSeverity::Info),
+ })];
+ events.push(request_metric(route.algorithm_name(), metadata.clone()));
+ if let Err(error) = route.check_caller_format(inbound) {
+ self.error_mark(&mut events, "caller_format", None);
+ return Execution {
+ result: Err(format!("Switchyard caller format is incompatible: {error}")),
+ events,
+ };
+ }
+ let observations = Arc::new(Mutex::new(Vec::new()));
+ let observed = Arc::clone(&observations);
+ let observer: RunObserver = Arc::new(move |observation| {
+ observed
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .push(observation);
+ });
+ match route.execute(request, Some(observer)).await {
+ Ok(output) => {
+ self.emit_observations(&mut events, take_observations(&observations), &metadata);
+ events.push(RoutingEvent::Mark(RoutingMark {
+ name: "switchyard.routing.decision".into(),
+ data: json!({
+ "algorithm": route.algorithm_name(),
+ "selected_model": output.selected_model,
+ }),
+ metadata,
+ severity: Some(LogSeverity::Info),
+ }));
+ Execution {
+ result: Ok(output.response),
+ events,
+ }
+ }
+ Err(error) => {
+ self.emit_observations(&mut events, take_observations(&observations), &metadata);
+ self.route_execution_error_mark(
+ &mut events,
+ &error.execution_error_summary(),
+ None,
+ );
+ Execution {
+ result: Err("Switchyard route execution failed".into()),
+ events,
+ }
+ }
+ }
+ }
+
+ fn route(&self, request: &Request) -> Option<&Route> {
+ request
+ .llm_request
+ .model
+ .as_deref()
+ .and_then(|model| self.runner.route(model))
+ }
+
+ fn emit_observations(
+ &self,
+ events: &mut Vec<RoutingEvent>,
+ observations: Vec<RunObservation>,
+ metadata: &Json,
+ ) {
+ let mut call_index = 0;
+ for observation in observations {
+ match observation {
+ RunObservation::LlmCall(call) => {
+ call_index += 1;
+ self.routing_call_events(events, call, call_index, metadata);
+ }
+ RunObservation::RoutingOverhead(duration) => {
+ let latency_ms = duration.as_secs_f64() * 1_000.0;
+ events.push(RoutingEvent::Mark(RoutingMark {
+ name: "switchyard.routing.overhead".into(),
+ data: json!({"latency_ms": latency_ms}),
+ metadata: metadata.clone(),
+ severity: Some(LogSeverity::Info),
+ }));
+ events.push(routing_overhead_metric(latency_ms, metadata.clone()));
+ }
+ RunObservation::AnswerCall(call) => {
+ events.extend(token_usage_metrics("answer", &call, metadata));
+ }
+ }
+ }
+ }
+
+ fn routing_call_events(
+ &self,
+ events: &mut Vec<RoutingEvent>,
+ call: LlmCallObservation,
+ call_index: usize,
+ metadata: &Json,
+ ) {
+ let outcome = if call.is_success { "ok" } else { "error" };
+ let latency_ms = call.duration.as_secs_f64() * 1_000.0;
+ let token_metrics = token_usage_metrics("routing", &call, metadata);
+ events.push(RoutingEvent::Mark(RoutingMark {
+ name: "switchyard.routing.llm_call".into(),
+ data: json!({
+ "call_index": call_index,
+ "selected_model": call.selected_model.as_str(),
+ "call_role": "routing",
+ "outcome": outcome,
+ "latency_ms": latency_ms,
+ }),
+ metadata: metadata.clone(),
+ severity: Some(LogSeverity::Debug),
+ }));
+ events.extend(routing_call_metrics(outcome, latency_ms, metadata.clone()));
+ events.extend(token_metrics);
+ }
+
+ fn error_mark(
+ &self,
+ events: &mut Vec<RoutingEvent>,
+ failure_kind: &str,
+ metadata: Option<&Json>,
+ ) {
+ let metadata = metadata
+ .cloned()
+ .unwrap_or_else(|| event_metadata(events).unwrap_or_else(|| Json::Object(Map::new())));
+ events.push(RoutingEvent::Mark(RoutingMark {
+ name: "switchyard.routing.error".into(),
+ data: json!({"failure_kind": failure_kind}),
+ metadata: metadata.clone(),
+ severity: Some(LogSeverity::Error),
+ }));
+ events.push(failure_metric(failure_kind, None, None, None, metadata));
+ }
+
+ fn route_execution_error_mark(
+ &self,
+ events: &mut Vec<RoutingEvent>,
+ summary: &RouteErrorSummary,
+ metadata: Option<&Json>,
+ ) {
+ let metadata = metadata
+ .cloned()
+ .unwrap_or_else(|| event_metadata(events).unwrap_or_else(|| Json::Object(Map::new())));
+ events.extend(route_execution_error_events(summary, metadata));
+ }
+}
+
+pub(crate) fn emit_events(runtime: &PluginRuntime, events: Vec<RoutingEvent>) {
+ for event in events {
+ emit_event(runtime, event);
+ }
+}
+
+pub(crate) fn emit_event(runtime: &PluginRuntime, event: RoutingEvent) {
+ let result = match event {
+ RoutingEvent::Mark(mark) => runtime
+ .emit_mark_with_options(
+ &mark.name,
+ Some(&mark.data),
+ Some(&mark.metadata),
+ None,
+ mark.severity,
+ )
+ .map_err(|error| ("routing mark", mark.name, error)),
+ RoutingEvent::Metric(metric) => runtime
+ .emit_metric(&metric.name, metric.measurements, Some(&metric.metadata))
+ .map_err(|error| ("routing metric", metric.name, error)),
+ };
+ if let Err((kind, name, error)) = result {
+ eprintln!("Switchyard could not emit {kind} {name:?}: {error}");
+ }
+}
+
+fn take_observations(observations: &Mutex<Vec<RunObservation>>) -> Vec<RunObservation> {
+ std::mem::take(
+ &mut *observations
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner()),
+ )
+}
+
+fn finalize_buffered_response(
+ translation_engine: &TranslationEngine,
+ inbound: WireFormat,
+ response: Response,
+ request_extensions: &ProviderExtensions,
+) -> Result<Json, String> {
+ let LlmResponse::Agg(response) = response.llm_response else {
+ return Err("Switchyard returned a stream for a buffered request".into());
+ };
+ translation::encode_response(translation_engine, inbound, &response, request_extensions)
+}
+
+fn returned_events(
+ response: Response,
+ inbound: WireFormat,
+ request_extensions: &ProviderExtensions,
+ metadata: Json,
+ emit_event: RoutingEventEmitter,
+) -> Result<ReturnedEventStream, String> {
+ let served_model = response.served_model().cloned();
+ let (chunks, observe_stream_usage) = match response.llm_response {
+ LlmResponse::Agg(response) => (response.into_stream(), false),
+ LlmResponse::Stream(chunks) => (chunks, true),
+ };
+ let mut latest_usage = None;
+ let mut terminal_seen = false;
+ let mut usage_emitted = false;
+ let mut stream_failed = false;
+ let mut failure_emitted = false;
+ let chunks = Box::pin(chunks.map(move |item| {
+ let in_band_error = item
+ .as_ref()
+ .ok()
+ .and_then(|event| normalized_stream_error(event.normalized()));
+ let stream_error = in_band_error.as_ref().or_else(|| item.as_ref().err());
+ stream_failed |= stream_error.is_some();
+ if let Some(error) = stream_error
+ && !failure_emitted
+ {
+ for event in route_execution_error_events(
+ &stream_error_summary(error, served_model.as_ref()),
+ metadata.clone(),
+ ) {
+ emit_event(event);
+ }
+ failure_emitted = true;
+ }
+ if observe_stream_usage && !stream_failed && !usage_emitted {
+ if let Ok(event) = &item {
+ for chunk in event.normalized() {
+ match chunk {
+ LlmResponseChunk::Usage(usage) => latest_usage = Some(usage.clone()),
+ LlmResponseChunk::MessageStop { .. } => terminal_seen = true,
+ _ => {}
+ }
+ }
+ }
+ if terminal_seen
+ && let (Some(model), Some(usage)) = (served_model.as_ref(), latest_usage.as_ref())
+ {
+ for event in usage_metrics("answer", model.as_str(), usage, &metadata) {
+ emit_event(event);
+ }
+ usage_emitted = true;
+ }
+ }
+ item
+ }));
+ let events = encode_stream_with_extensions(chunks, inbound, None, request_extensions)
+ .map_err(|error| format!("Switchyard response stream setup failed: {error}"))?;
+ Ok(Box::pin(
+ events.map(|item| item.map_err(relay_stream_error)),
+ ))
+}
+
+/// Produces a telemetry-only error summary source for a normalized in-band failure.
+///
+/// The translation layer retains the original provider error event for client delivery;
+/// this conversion is only used to classify the failure without putting its message in marks.
+fn normalized_stream_error(chunks: &[LlmResponseChunk]) -> Option<LlmClientError> {
+ chunks.iter().find_map(|chunk| match chunk {
+ LlmResponseChunk::DecodeError { message } => {
+ Some(LlmClientError::ResponseTranslation(message.clone()))
+ }
+ LlmResponseChunk::StreamError { message } => Some(LlmClientError::UpstreamHttp {
+ status: StatusCode::BAD_GATEWAY,
+ body: message.clone(),
+ }),
+ _ => None,
+ })
+}
+
+/// Converts translation errors to Relay's string-only stream-error boundary.
+///
+/// `LlmStreamError::Upstream` retains a provider-shaped JSON value so its caller can
+/// make protocol decisions, but neither that value nor a client error's source message
+/// is safe for Relay's outer observability scope. Preserve the error result while
+/// exposing only a stable failure class.
+fn relay_stream_error(error: LlmStreamError) -> String {
+ match error {
+ LlmStreamError::Upstream(_) => "Switchyard upstream stream error".into(),
+ LlmStreamError::Client(error) => format!(
+ "Switchyard stream error: {}",
+ stream_error_summary(&error, None).kind.as_str()
+ ),
+ }
+}
+
+fn route_execution_error_mark(summary: &RouteErrorSummary, metadata: Json) -> RoutingMark {
+ RoutingMark {
+ name: "switchyard.routing.error".into(),
+ data: json!({
+ "failure_kind": "route_execution",
+ "category": summary.kind.as_str(),
+ "phase": summary.phase.as_str(),
+ "upstream_status": summary.upstream_status,
+ "target": summary.target.as_ref().map(|target| target.as_str()),
+ }),
+ metadata,
+ severity: Some(LogSeverity::Error),
+ }
+}
+
+fn route_execution_error_events(summary: &RouteErrorSummary, metadata: Json) -> Vec<RoutingEvent> {
+ vec![
+ RoutingEvent::Mark(route_execution_error_mark(summary, metadata.clone())),
+ failure_metric(
+ "route_execution",
+ Some(summary.kind.as_str()),
+ Some(summary.phase.as_str()),
+ summary.upstream_status,
+ metadata,
+ ),
+ ]
+}
+
+fn event_metadata(events: &[RoutingEvent]) -> Option<Json> {
+ events.iter().find_map(|event| match event {
+ RoutingEvent::Mark(mark) => Some(mark.metadata.clone()),
+ RoutingEvent::Metric(_) => None,
+ })
+}
+
+fn request_metric(algorithm: &str, metadata: Json) -> RoutingEvent {
+ counter_metric(
+ "switchyard.routing.requests",
+ "Requests managed by Switchyard routing.",
+ json!({"algorithm": algorithm}),
+ metadata,
+ )
+}
+
+fn routing_call_metrics(outcome: &str, latency_ms: f64, metadata: Json) -> [RoutingEvent; 2] {
+ let attributes = json!({"outcome": outcome});
+ [
+ counter_metric(
+ "switchyard.routing.llm_calls",
+ "Switchyard model calls made while routing.",
+ attributes.clone(),
+ metadata.clone(),
+ ),
+ histogram_metric(
+ "switchyard.routing.llm_call.duration",
+ "Duration of Switchyard model calls made while routing.",
+ latency_ms,
+ attributes,
+ metadata,
+ ),
+ ]
+}
+
+fn routing_overhead_metric(latency_ms: f64, metadata: Json) -> RoutingEvent {
+ histogram_metric(
+ "switchyard.routing.overhead",
+ "Time needed to produce the Switchyard routing outcome, including routing model calls.",
+ latency_ms,
+ json!({}),
+ metadata,
+ )
+}
+
+fn token_usage_metrics(
+ call_role: &str,
+ call: &LlmCallObservation,
+ metadata: &Json,
+) -> Vec<RoutingEvent> {
+ let Some(usage) = call.usage.as_ref() else {
+ return Vec::new();
+ };
+ usage_metrics(call_role, call.selected_model.as_str(), usage, metadata)
+}
+
+fn usage_metrics(
+ call_role: &str,
+ target_model: &str,
+ usage: &Usage,
+ metadata: &Json,
+) -> Vec<RoutingEvent> {
+ [
+ ("input", usage.input_tokens),
+ ("cached_input", usage.cached_input_tokens()),
+ ("cache_creation_input", usage.cache_creation_input_tokens()),
+ ("output", usage.output_tokens),
+ ("reasoning", usage.reasoning_tokens),
+ ("total", usage.total_tokens),
+ ]
+ .into_iter()
+ .filter_map(|(token_type, value)| {
+ value.map(|value| {
+ metric(
+ MetricDescriptor {
+ name: "switchyard.routing.llm_tokens",
+ kind: MetricKind::Counter,
+ value_type: MetricValueType::U64,
+ unit: Some("{token}"),
+ description: "Normalized tokens used by Switchyard model calls.",
+ },
+ json!(value),
+ json!({
+ "call_role": call_role,
+ "target_model": target_model,
+ "token_type": token_type,
+ }),
+ metadata.clone(),
+ )
+ })
+ })
+ .collect()
+}
+
+fn failure_metric(
+ failure_kind: &str,
+ category: Option<&str>,
+ phase: Option<&str>,
+ upstream_status: Option<u16>,
+ metadata: Json,
+) -> RoutingEvent {
+ let mut attributes = Map::new();
+ attributes.insert("failure_kind".into(), Json::String(failure_kind.into()));
+ if let Some(category) = category {
+ attributes.insert("category".into(), Json::String(category.into()));
+ }
+ if let Some(phase) = phase {
+ attributes.insert("phase".into(), Json::String(phase.into()));
+ }
+ if let Some(upstream_status) = upstream_status {
+ attributes.insert("upstream_status".into(), Json::from(upstream_status));
+ }
+ counter_metric(
+ "switchyard.routing.failures",
+ "Terminal Switchyard routing failures.",
+ Json::Object(attributes),
+ metadata,
+ )
+}
+
+fn counter_metric(name: &str, description: &str, attributes: Json, metadata: Json) -> RoutingEvent {
+ metric(
+ MetricDescriptor {
+ name,
+ kind: MetricKind::Counter,
+ value_type: MetricValueType::U64,
+ unit: Some("{event}"),
+ description,
+ },
+ json!(1),
+ attributes,
+ metadata,
+ )
+}
+
+fn histogram_metric(
+ name: &str,
+ description: &str,
+ value: f64,
+ attributes: Json,
+ metadata: Json,
+) -> RoutingEvent {
+ metric(
+ MetricDescriptor {
+ name,
+ kind: MetricKind::Histogram,
+ value_type: MetricValueType::F64,
+ unit: Some("ms"),
+ description,
+ },
+ json!(value),
+ attributes,
+ metadata,
+ )
+}
+
+fn metric(
+ descriptor: MetricDescriptor<'_>,
+ value: Json,
+ attributes: Json,
+ metadata: Json,
+) -> RoutingEvent {
+ RoutingEvent::Metric(RoutingMetric {
+ name: descriptor.name.into(),
+ measurements: vec![MetricMeasurement {
+ name: descriptor.name.into(),
+ kind: descriptor.kind,
+ value_type: descriptor.value_type,
+ value,
+ unit: descriptor.unit.map(Into::into),
+ description: Some(descriptor.description.into()),
+ attributes: Some(attributes),
+ boundaries: None,
+ }],
+ metadata,
+ })
+}
+
+fn string_headers(headers: &Map<String, Json>) -> http::HeaderMap {
+ let mut parsed = http::HeaderMap::with_capacity(headers.len());
+ for (name, value) in headers {
+ let Some(value) = value.as_str() else {
+ continue;
+ };
+ let (Ok(name), Ok(value)) = (
+ http::HeaderName::from_bytes(name.as_bytes()),
+ http::HeaderValue::from_str(value),
+ ) else {
+ continue;
+ };
+ parsed.insert(name, value);
+ }
+ parsed
+}
+
+fn identity_metadata(metadata: Option<&Metadata>) -> Json {
+ json!({
+ "session_id": metadata.and_then(|value| value.session_id.as_deref()),
+ "agent_id": metadata.and_then(|value| value.agent_id.as_deref()),
+ "parent_agent_id": metadata.and_then(|value| value.parent_agent_id.as_deref()),
+ "task_id": metadata.and_then(|value| value.task_id.as_deref()),
+ "turn_id": metadata.and_then(|value| value.turn_id.as_deref()),
+ "correlation_id": metadata.and_then(|value| value.correlation_id.as_deref()),
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::{BTreeMap, HashMap};
+
+ use switchyard_llm_client::ClientRouter;
+ use switchyard_protocol::{LlmClientError, LlmResponseStreamEvent, ModelId, Usage};
+ use switchyard_runner::{AlgorithmSpec, ModelCapabilities, RunnerError};
+ use wiremock::matchers::{body_partial_json, method, path};
+ use wiremock::{Mock, MockServer, ResponseTemplate};
+
+ use super::*;
+
+ fn runtime_for(model: &str) -> SwitchyardRuntime {
+ let algorithm = AlgorithmSpec::Noop {}
+ .build("relay", &BTreeMap::new())
+ .expect("noop route should build");
+ let route = Route::new(
+ algorithm,
+ ClientRouter::new(HashMap::new()),
+ None,
+ ModelCapabilities::default(),
+ None,
+ None,
+ Vec::new(),
+ );
+ SwitchyardRuntime {
+ runner: Runner::new(vec![(ModelId::from(model), route)]),
+ translation: TranslationEngine::default(),
+ }
+ }
+
+ fn runtime_for_target(format: &str, base_url: &str) -> SwitchyardRuntime {
+ let deployment = json!({
+ "schema_version": 1,
+ "llm_clients": {
+ "target": {
+ "format": format,
+ "base_url": base_url,
+ }
+ },
+ "targets": {
+ "default": {
+ "id": "target/model",
+ "llm_client": "target",
+ }
+ },
+ "routes": {
+ "default": {
+ "id": "switchyard/default",
+ "type": "passthrough",
+ "target": "default",
+ }
+ }
+ });
+ SwitchyardRuntime::new(crate::config::SwitchyardConfig {
+ priority: 0,
+ switchyard_config_path: None,
+ switchyard_config: Some(deployment.as_object().unwrap().clone()),
+ })
+ .expect("cross-format runtime should load")
+ }
+
+ fn namespaced_responses_request() -> RelayRequest {
+ RelayRequest {
+ headers: Map::new(),
+ content: json!({
+ "model": "switchyard/default",
+ "input": "Search the docs",
+ "tools": [{
+ "type": "namespace",
+ "name": "mcp__docs",
+ "tools": [{
+ "type": "function",
+ "name": "search",
+ "parameters": {
+ "type": "object",
+ "properties": {"query": {"type": "string"}},
+ "required": ["query"],
+ },
+ }],
+ }],
+ }),
+ }
+ }
+
+ #[test]
+ fn only_configured_route_models_are_managed() {
+ let runtime = runtime_for("switchyard");
+ assert!(runtime.manages_model("switchyard"));
+ assert!(!runtime.manages_model("other"));
+ }
+
+ #[tokio::test]
+ async fn responses_requests_use_the_configured_target_format() {
+ for (format, endpoint, upstream_response) in [
+ (
+ "openai_chat",
+ "/v1/chat/completions",
+ json!({
+ "id": "chatcmpl_test",
+ "object": "chat.completion",
+ "model": "target/model",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+ }),
+ ),
+ (
+ "anthropic_messages",
+ "/v1/messages",
+ json!({
+ "id": "msg_test",
+ "type": "message",
+ "role": "assistant",
+ "model": "target/model",
+ "content": [{"type": "text", "text": "ok"}],
+ "stop_reason": "end_turn",
+ "usage": {"input_tokens": 1, "output_tokens": 1},
+ }),
+ ),
+ ] {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path(endpoint))
+ .and(body_partial_json(json!({"model": "target/model"})))
+ .respond_with(ResponseTemplate::new(200).set_body_json(upstream_response))
+ .expect(1)
+ .mount(&server)
+ .await;
+ let runtime = runtime_for_target(format, &format!("{}/v1", server.uri()));
+ let request = RelayRequest {
+ headers: Map::new(),
+ content: json!({
+ "model": "switchyard/default",
+ "input": "hello",
+ }),
+ };
+ let decoded = runtime
+ .decode_request(WireFormat::OpenAiResponses, request, false)
+ .expect("Responses request should decode");
+ assert_eq!(
+ decoded
+ .metadata
+ .as_ref()
+ .and_then(|metadata| metadata.wire_format),
+ None
+ );
+
+ let execution = runtime
+ .execute_buffered(WireFormat::OpenAiResponses, decoded)
+ .await;
+ let response = execution.result.expect("target call should succeed");
+ assert_eq!(response["object"], "response");
+ assert_eq!(response["model"], "target/model");
+ }
+ }
+
+ #[tokio::test]
+ async fn buffered_responses_restore_codex_tool_namespaces() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .and(body_partial_json(json!({"model": "target/model"})))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "id": "chatcmpl_tool",
+ "object": "chat.completion",
+ "model": "target/model",
+ "choices": [{
+ "index": 0,
+ "message": {
+ "role": "assistant",
+ "content": null,
+ "tool_calls": [{
+ "id": "call_search",
+ "type": "function",
+ "function": {
+ "name": "mcp__docs__search",
+ "arguments": "{\"query\":\"routing\"}",
+ },
+ }],
+ },
+ "finish_reason": "tool_calls",
+ }],
+ "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ let runtime = runtime_for_target("openai_chat", &format!("{}/v1", server.uri()));
+ let request = runtime
+ .decode_request(
+ WireFormat::OpenAiResponses,
+ namespaced_responses_request(),
+ false,
+ )
+ .expect("namespaced Responses request should decode");
+
+ let response = runtime
+ .execute_buffered(WireFormat::OpenAiResponses, request)
+ .await
+ .result
+ .expect("buffered target call should succeed");
+ let tool_call = response["output"]
+ .as_array()
+ .and_then(|output| output.iter().find(|item| item["type"] == "function_call"))
+ .expect("Responses output should contain a function call");
+ assert_eq!(tool_call["name"], "search");
+ assert_eq!(tool_call["namespace"], "mcp__docs");
+ }
+
+ #[tokio::test]
+ async fn streaming_responses_restore_codex_tool_namespaces() {
+ let server = MockServer::start().await;
+ let sse = concat!(
+ "data: {\"id\":\"chatcmpl_tool\",\"object\":\"chat.completion.chunk\",\"model\":\"target/model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_search\",\"type\":\"function\",\"function\":{\"name\":\"mcp__docs__search\",\"arguments\":\"{\\\"query\\\":\\\"routing\\\"}\"}}]},\"finish_reason\":null}]}\n\n",
+ "data: {\"id\":\"chatcmpl_tool\",\"object\":\"chat.completion.chunk\",\"model\":\"target/model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n",
+ "data: [DONE]\n\n",
+ );
+ Mock::given(method("POST"))
+ .and(path("/v1/chat/completions"))
+ .and(body_partial_json(
+ json!({"model": "target/model", "stream": true}),
+ ))
+ .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))
+ .expect(1)
+ .mount(&server)
+ .await;
+ let runtime = runtime_for_target("openai_chat", &format!("{}/v1", server.uri()));
+ let request = runtime
+ .decode_request(
+ WireFormat::OpenAiResponses,
+ namespaced_responses_request(),
+ true,
+ )
+ .expect("namespaced streaming Responses request should decode");
+
+ let stream = runtime
+ .execute_stream(WireFormat::OpenAiResponses, request, Arc::new(|_| {}))
+ .await
+ .result
+ .expect("streaming target call should succeed");
+ let events = stream
+ .collect::<Vec<_>>()
+ .await
+ .into_iter()
+ .collect::<Result<Vec<_>, _>>()
+ .expect("response stream should encode");
+
+ for event_type in ["response.output_item.added", "response.output_item.done"] {
+ let item = events
+ .iter()
+ .find(|event| event["type"] == event_type)
+ .map(|event| &event["item"])
+ .unwrap_or_else(|| panic!("stream produced no {event_type}"));
+ assert_eq!(item["name"], "search", "{event_type}");
+ assert_eq!(item["namespace"], "mcp__docs", "{event_type}");
+ }
+ let completed = events
+ .iter()
+ .find(|event| event["type"] == "response.completed")
+ .expect("stream produced no response.completed event");
+ assert_eq!(completed["response"]["output"][0]["name"], "search");
+ assert_eq!(completed["response"]["output"][0]["namespace"], "mcp__docs");
+ }
+
+ #[test]
+ fn execution_failure_mark_uses_the_safe_runner_summary() {
+ let secret = "provider response body";
+ let error = RunnerError::Client(LlmClientError::ContextWindowExceeded {
+ model: ModelId::from("weak"),
+ message: secret.into(),
+ });
+
+ let mark = route_execution_error_mark(
+ &error.execution_error_summary(),
+ json!({"session_id": "session"}),
+ );
+
+ assert_eq!(mark.name, "switchyard.routing.error");
+ assert_eq!(mark.data["failure_kind"], "route_execution");
+ assert_eq!(mark.data["category"], "context_window_exceeded");
+ assert_eq!(mark.data["phase"], "before_response");
+ assert_eq!(mark.data["target"], "weak");
+ assert_eq!(mark.data["upstream_status"], Json::Null);
+ assert_eq!(mark.severity, Some(LogSeverity::Error));
+ assert!(!mark.data.to_string().contains(secret));
+ }
+
+ #[test]
+ fn routing_observations_emit_debug_marks_and_metrics() {
+ let runtime = runtime_for("switchyard");
+ let mut events = Vec::new();
+ runtime.emit_observations(
+ &mut events,
+ vec![
+ RunObservation::LlmCall(LlmCallObservation {
+ selected_model: ModelId::from("routing-model"),
+ is_success: false,
+ duration: std::time::Duration::from_millis(12),
+ usage: Some(Usage {
+ input_tokens: Some(4),
+ ..Usage::default()
+ }),
+ }),
+ RunObservation::RoutingOverhead(std::time::Duration::from_millis(3)),
+ ],
+ &json!({"session_id": "session"}),
+ );
+
+ assert_eq!(events.len(), 6);
+ let RoutingEvent::Mark(call_mark) = &events[0] else {
+ panic!("first event should be the routing call mark");
+ };
+ assert_eq!(call_mark.name, "switchyard.routing.llm_call");
+ assert_eq!(call_mark.severity, Some(LogSeverity::Debug));
+ assert_eq!(call_mark.data["outcome"], "error");
+ assert!(call_mark.data.get("usage").is_none());
+
+ let RoutingEvent::Metric(call_count) = &events[1] else {
+ panic!("second event should be the routing call counter");
+ };
+ assert_eq!(call_count.name, "switchyard.routing.llm_calls");
+ assert_eq!(call_count.measurements[0].kind, MetricKind::Counter);
+ assert_eq!(
+ call_count.measurements[0].attributes,
+ Some(json!({"outcome": "error"}))
+ );
+
+ let RoutingEvent::Metric(call_duration) = &events[2] else {
+ panic!("third event should be the routing call histogram");
+ };
+ assert_eq!(call_duration.name, "switchyard.routing.llm_call.duration");
+ assert_eq!(call_duration.measurements[0].kind, MetricKind::Histogram);
+ assert_eq!(call_duration.measurements[0].value, json!(12.0));
+
+ let RoutingEvent::Metric(tokens) = &events[3] else {
+ panic!("fourth event should be the routing token counter");
+ };
+ assert_eq!(tokens.name, "switchyard.routing.llm_tokens");
+
+ let RoutingEvent::Mark(overhead_mark) = &events[4] else {
+ panic!("fifth event should be the routing overhead mark");
+ };
+ assert_eq!(overhead_mark.severity, Some(LogSeverity::Info));
+
+ let RoutingEvent::Metric(overhead) = &events[5] else {
+ panic!("sixth event should be the routing overhead histogram");
+ };
+ assert_eq!(overhead.name, "switchyard.routing.overhead");
+ assert_eq!(overhead.measurements[0].attributes, Some(json!({})));
+ }
+
+ #[test]
+ fn request_and_failure_metrics_use_bounded_attributes() {
+ let RoutingEvent::Metric(request) = request_metric("stage_router", json!({})) else {
+ panic!("request should be a metric");
+ };
+ assert_eq!(request.name, "switchyard.routing.requests");
+ assert_eq!(
+ request.measurements[0].attributes,
+ Some(json!({"algorithm": "stage_router"}))
+ );
+
+ let RoutingEvent::Metric(failure) = failure_metric(
+ "route_execution",
+ Some("upstream_http"),
+ Some("before_response"),
+ Some(503),
+ json!({}),
+ ) else {
+ panic!("failure should be a metric");
+ };
+ assert_eq!(failure.name, "switchyard.routing.failures");
+ assert_eq!(
+ failure.measurements[0].attributes,
+ Some(json!({
+ "failure_kind": "route_execution",
+ "category": "upstream_http",
+ "phase": "before_response",
+ "upstream_status": 503,
+ }))
+ );
+ }
+
+ #[test]
+ fn token_usage_metrics_distinguish_routing_and_answer_targets() {
+ let call = LlmCallObservation {
+ selected_model: ModelId::from("judge-model"),
+ is_success: true,
+ duration: std::time::Duration::from_millis(1),
+ usage: Some(Usage {
+ input_tokens: Some(11),
+ cache: Usage::cache_details(Some(3), Some(2)),
+ output_tokens: Some(7),
+ total_tokens: Some(23),
+ reasoning_tokens: Some(5),
+ }),
+ };
+
+ let routing = token_usage_metrics("routing", &call, &json!({"session_id": "session"}));
+ assert_eq!(routing.len(), 6);
+ for event in &routing {
+ let RoutingEvent::Metric(metric) = event else {
+ panic!("token usage should be emitted as a metric");
+ };
+ assert_eq!(metric.name, "switchyard.routing.llm_tokens");
+ assert_eq!(metric.measurements[0].kind, MetricKind::Counter);
+ assert_eq!(metric.measurements[0].unit.as_deref(), Some("{token}"));
+ assert_eq!(
+ metric.measurements[0].attributes.as_ref().unwrap()["call_role"],
+ "routing"
+ );
+ assert_eq!(
+ metric.measurements[0].attributes.as_ref().unwrap()["target_model"],
+ "judge-model"
+ );
+ }
+ let token_values = routing
+ .iter()
+ .map(|event| {
+ let RoutingEvent::Metric(metric) = event else {
+ panic!("token usage should be emitted as a metric");
+ };
+ metric.measurements[0].value.clone()
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(
+ token_values,
+ vec![json!(11), json!(3), json!(2), json!(7), json!(5), json!(23)]
+ );
+
+ let answer = token_usage_metrics("answer", &call, &json!({}));
+ let RoutingEvent::Metric(metric) = &answer[0] else {
+ panic!("answer usage should be emitted as a metric");
+ };
+ assert_eq!(
+ metric.measurements[0].attributes.as_ref().unwrap()["call_role"],
+ "answer"
+ );
+ }
+
+ #[test]
+ fn answer_observations_emit_token_metrics_without_answer_logs() {
+ let runtime = runtime_for("switchyard");
+ let mut events = Vec::new();
+ runtime.emit_observations(
+ &mut events,
+ vec![RunObservation::AnswerCall(LlmCallObservation {
+ selected_model: ModelId::from("selected-target"),
+ is_success: true,
+ duration: std::time::Duration::from_millis(2),
+ usage: Some(Usage {
+ output_tokens: Some(9),
+ ..Usage::default()
+ }),
+ })],
+ &json!({}),
+ );
+
+ assert_eq!(events.len(), 1);
+ let RoutingEvent::Metric(metric) = &events[0] else {
+ panic!("answer observation should only emit a token metric");
+ };
+ assert_eq!(metric.name, "switchyard.routing.llm_tokens");
+ assert_eq!(
+ metric.measurements[0].attributes,
+ Some(json!({
+ "call_role": "answer",
+ "target_model": "selected-target",
+ "token_type": "output",
+ }))
+ );
+ }
+
+ #[tokio::test]
+ async fn stream_usage_emits_answer_token_metrics() {
+ let response = Response {
+ llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([
+ Ok(LlmResponseStreamEvent::new(vec![
+ LlmResponseChunk::MessageStop { reason: None },
+ ])),
+ Ok(LlmResponseStreamEvent::new(vec![LlmResponseChunk::Usage(
+ Usage {
+ input_tokens: Some(10),
+ output_tokens: Some(3),
+ total_tokens: Some(13),
+ ..Usage::default()
+ },
+ )])),
+ ]))),
+ metadata: Some(Metadata {
+ served_model: Some(ModelId::from("selected-target")),
+ ..Default::default()
+ }),
+ };
+ let captured = Arc::new(Mutex::new(Vec::new()));
+ let emitted = Arc::clone(&captured);
+ let mut stream = returned_events(
+ response,
+ WireFormat::OpenAiChat,
+ &ProviderExtensions::default(),
+ json!({"session_id": "session"}),
+ Arc::new(move |event| emitted.lock().unwrap().push(event)),
+ )
+ .expect("stream setup should succeed");
+
+ assert!(captured.lock().unwrap().is_empty());
+ assert!(stream.next().await.expect("encoded stream event").is_ok());
+ assert!(captured.lock().unwrap().is_empty());
+ assert!(stream.next().await.expect("encoded usage event").is_ok());
+
+ let events = captured.lock().unwrap();
+ assert_eq!(events.len(), 3);
+ let token_types = events
+ .iter()
+ .map(|event| {
+ let RoutingEvent::Metric(metric) = event else {
+ panic!("stream usage should emit only metrics");
+ };
+ assert_eq!(metric.name, "switchyard.routing.llm_tokens");
+ assert_eq!(metric.metadata, json!({"session_id": "session"}));
+ let attributes = metric.measurements[0].attributes.as_ref().unwrap();
+ assert_eq!(attributes["call_role"], "answer");
+ assert_eq!(attributes["target_model"], "selected-target");
+ attributes["token_type"].as_str().unwrap().to_string()
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(token_types, ["input", "output", "total"]);
+ }
+
+ #[tokio::test]
+ async fn aggregate_response_stream_does_not_reemit_usage_metrics() {
+ let mut aggregate =
+ switchyard_protocol::text_response(Some("selected-target".into()), "ok");
+ aggregate.usage = Usage {
+ input_tokens: Some(10),
+ output_tokens: Some(3),
+ total_tokens: Some(13),
+ ..Usage::default()
+ };
+ let response = Response {
+ llm_response: LlmResponse::Agg(aggregate),
+ metadata: Some(Metadata {
+ served_model: Some(ModelId::from("selected-target")),
+ ..Default::default()
+ }),
+ };
+ let captured = Arc::new(Mutex::new(Vec::new()));
+ let emitted = Arc::clone(&captured);
+ let stream = returned_events(
+ response,
+ WireFormat::OpenAiChat,
+ &ProviderExtensions::default(),
+ json!({}),
+ Arc::new(move |event| emitted.lock().unwrap().push(event)),
+ )
+ .expect("stream setup should succeed");
+
+ let events = stream.collect::<Vec<_>>().await;
+ assert!(events.iter().all(Result::is_ok));
+ assert!(captured.lock().unwrap().is_empty());
+ }
+
+ #[tokio::test]
+ async fn stream_failure_emits_a_safe_failure_mark() {
+ let secret = "provider response body";
+ let response = Response {
+ llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([Err::<
+ LlmResponseStreamEvent,
+ LlmClientError,
+ >(
+ LlmClientError::ContextWindowExceeded {
+ model: ModelId::from("weak"),
+ message: secret.into(),
+ },
+ )]))),
+ metadata: Some(Metadata {
+ served_model: Some(ModelId::from("strong")),
+ ..Default::default()
+ }),
+ };
+ let captured = Arc::new(Mutex::new(Vec::new()));
+ let emitted = Arc::clone(&captured);
+ let stream = returned_events(
+ response,
+ WireFormat::OpenAiChat,
+ &ProviderExtensions::default(),
+ json!({"session_id": "session"}),
+ Arc::new(move |mark| emitted.lock().unwrap().push(mark)),
+ )
+ .expect("stream setup should succeed");
+
+ let events = stream.collect::<Vec<_>>().await;
+ assert!(events[0].is_err());
+ assert!(
+ !events[0]
+ .as_ref()
+ .expect_err("stream should fail")
+ .contains(secret)
+ );
+ let events = captured.lock().unwrap();
+ assert_eq!(events.len(), 2);
+ let RoutingEvent::Mark(mark) = &events[0] else {
+ panic!("first event should be the safe failure mark");
+ };
+ assert_eq!(mark.data["category"], "context_window_exceeded");
+ assert_eq!(mark.data["phase"], "during_stream");
+ assert_eq!(mark.data["target"], "strong");
+ assert_eq!(mark.severity, Some(LogSeverity::Error));
+ assert!(!mark.data.to_string().contains(secret));
+ let RoutingEvent::Metric(metric) = &events[1] else {
+ panic!("second event should be the failure counter");
+ };
+ assert_eq!(metric.name, "switchyard.routing.failures");
+ assert_eq!(
+ metric.measurements[0].attributes,
+ Some(json!({
+ "failure_kind": "route_execution",
+ "category": "context_window_exceeded",
+ "phase": "during_stream",
+ }))
+ );
+ }
+
+ async fn assert_normalized_stream_failure(
+ chunk: LlmResponseChunk,
+ expected_category: &str,
+ expected_status: Option<u16>,
+ ) {
+ let secret = "provider response body";
+ let response = Response {
+ llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([Ok(
+ LlmResponseStreamEvent::new(vec![chunk]),
+ )]))),
+ metadata: Some(Metadata {
+ served_model: Some(ModelId::from("selected-target")),
+ ..Default::default()
+ }),
+ };
+ let captured = Arc::new(Mutex::new(Vec::new()));
+ let emitted = Arc::clone(&captured);
+ let stream = returned_events(
+ response,
+ WireFormat::OpenAiChat,
+ &ProviderExtensions::default(),
+ json!({"session_id": "session"}),
+ Arc::new(move |event| emitted.lock().unwrap().push(event)),
+ )
+ .expect("stream setup should succeed");
+
+ let client_events = stream.collect::<Vec<_>>().await;
+ assert_eq!(client_events.len(), 1);
+ assert!(client_events[0].is_err());
+ assert!(
+ !client_events[0]
+ .as_ref()
+ .expect_err("stream should fail")
+ .contains(secret)
+ );
+
+ let events = captured.lock().unwrap();
+ assert_eq!(events.len(), 2, "one mark and one counter should emit");
+ let RoutingEvent::Mark(mark) = &events[0] else {
+ panic!("first event should be the safe failure mark");
+ };
+ assert_eq!(mark.data["category"], expected_category);
+ assert_eq!(mark.data["phase"], "during_stream");
+ assert_eq!(
+ mark.data["upstream_status"],
+ expected_status.map_or(Json::Null, Json::from)
+ );
+ assert_eq!(mark.data["target"], "selected-target");
+ assert!(!mark.data.to_string().contains(secret));
+ let RoutingEvent::Metric(metric) = &events[1] else {
+ panic!("second event should be the failure counter");
+ };
+ assert_eq!(metric.name, "switchyard.routing.failures");
+ let mut expected_attributes = json!({
+ "failure_kind": "route_execution",
+ "category": expected_category,
+ "phase": "during_stream",
+ });
+ if let Some(status) = expected_status {
+ expected_attributes
+ .as_object_mut()
+ .expect("expected metric attributes should be an object")
+ .insert("upstream_status".into(), json!(status));
+ }
+ assert_eq!(metric.measurements[0].attributes, Some(expected_attributes));
+ }
+
+ #[tokio::test]
+ async fn normalized_stream_error_emits_failure_telemetry_once() {
+ assert_normalized_stream_failure(
+ LlmResponseChunk::StreamError {
+ message: "provider response body".into(),
+ },
+ "upstream_http",
+ Some(502),
+ )
+ .await;
+ }
+
+ #[tokio::test]
+ async fn normalized_decode_error_emits_failure_telemetry_once() {
+ assert_normalized_stream_failure(
+ LlmResponseChunk::DecodeError {
+ message: "provider response body".into(),
+ },
+ "response_translation",
+ None,
+ )
+ .await;
+ }
+}
diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs
new file mode 100644
index 000000000..b565d5b58
--- /dev/null
+++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs
@@ -0,0 +1,66 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use nemo_relay_plugin::LlmRequest as RelayRequest;
+use serde_json::Value as Json;
+use switchyard_protocol::{AggLlmResponse, LlmRequest, ProviderExtensions, WireFormat};
+use switchyard_translation::{
+ DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy,
+ TargetCapabilities, TranslationDiagnostic, TranslationEngine, TranslationPolicy,
+ UnknownFieldPolicy,
+};
+
+pub(crate) fn decode_request(
+ engine: &TranslationEngine,
+ protocol: WireFormat,
+ request: &RelayRequest,
+) -> Result<LlmRequest, String> {
+ let output = engine
+ .decode_request(protocol, &request.content, &policy())
+ .map_err(error)?;
+ safe(&output.diagnostics)?;
+ Ok(output.request)
+}
+
+pub(crate) fn encode_response(
+ engine: &TranslationEngine,
+ protocol: WireFormat,
+ response: &AggLlmResponse,
+ request_extensions: &ProviderExtensions,
+) -> Result<Json, String> {
+ let output = engine
+ .encode_response_with_extensions(protocol, response, request_extensions, &policy())
+ .map_err(error)?;
+ safe(&output.diagnostics)?;
+ Ok(output.body)
+}
+
+fn policy() -> TranslationPolicy {
+ TranslationPolicy {
+ unknown_field_policy: UnknownFieldPolicy::Preserve,
+ lossy_conversion_policy: LossyConversionPolicy::Reject,
+ deterministic_ids: DeterministicIdPolicy::GenerateStable {
+ prefix: "relay".into(),
+ },
+ preservation: PreservationPolicy::InMemory,
+ target_capabilities: TargetCapabilities::default(),
+ }
+}
+
+fn safe(diagnostics: &[TranslationDiagnostic]) -> Result<(), String> {
+ let unsafe_diagnostics = diagnostics
+ .iter()
+ .filter(|diagnostic| diagnostic.severity != DiagnosticSeverity::Info)
+ .collect::<Vec<_>>();
+ if unsafe_diagnostics.is_empty() {
+ Ok(())
+ } else {
+ Err(format!(
+ "Switchyard translation was not lossless: {unsafe_diagnostics:?}"
+ ))
+ }
+}
+
+fn error(error: switchyard_translation::TranslationError) -> String {
+ format!("Switchyard translation failed: {error}")
+}
diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs
index 9761a46af..abd30955c 100644
--- a/crates/switchyard-translation/src/engine.rs
+++ b/crates/switchyard-translation/src/engine.rs
@@ -19,7 +19,7 @@ use crate::codecs::stream::{
use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::FormatId;
-use crate::llm::{AggLlmResponse, LlmRequest};
+use crate::llm::{AggLlmResponse, LlmRequest, ProviderExtensions};
use crate::policy::TranslationPolicy;
/// Encoded translation result with any diagnostics emitted along the way.
@@ -201,6 +201,25 @@ impl TranslationEngine {
})
}
+ /// Encodes a neutral response IR while applying metadata retained from its request.
+ ///
+ /// Request extensions are used to restore caller-specific response details, such as
+ /// Codex tool namespaces, without exposing those details to the upstream provider.
+ pub fn encode_response_with_extensions(
+ &self,
+ target: impl Into<FormatId>,
+ response: &AggLlmResponse,
+ request_extensions: &ProviderExtensions,
+ policy: &TranslationPolicy,
+ ) -> Result<TranslationOutput> {
+ let mut output = self.encode_response(target, response, policy)?;
+ crate::codex_namespaces::restore_qualified_tool_names(
+ &mut output.body,
+ &crate::codex_namespaces::qualified_tool_origins(request_extensions),
+ );
+ Ok(output)
+ }
+
/// Translates a response body from source format to target format.
pub fn translate_response(
&self,
diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs
index a1cdd1d3c..061517e24 100644
--- a/crates/switchyard-translation/src/helpers.rs
+++ b/crates/switchyard-translation/src/helpers.rs
@@ -76,15 +76,16 @@ pub fn encode_aggregated_response_with_extensions(
request_extensions: &switchyard_protocol::ProviderExtensions,
) -> Result<Value> {
let mut body = DEFAULT_TRANSLATION_ENGINE
- .encode_response(wire_format, agg, &DEFAULT_TRANSLATION_POLICY)?
+ .encode_response_with_extensions(
+ wire_format,
+ agg,
+ request_extensions,
+ &DEFAULT_TRANSLATION_POLICY,
+ )?
.body;
if let (Some(model), Value::Object(object)) = (served_model, &mut body) {
object.insert("model".to_string(), Value::String(model.to_string()));
}
- crate::codex_namespaces::restore_qualified_tool_names(
- &mut body,
- &crate::codex_namespaces::qualified_tool_origins(request_extensions),
- );
Ok(body)
}
diff --git a/docs/index.md b/docs/index.md
index e68ed477b..9b8fb17f9 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -9,6 +9,7 @@ It supports OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages.
|---|---|---|
| Run Switchyard as a standalone proxy for API clients | Server Path | [Build and run the Rust server](getting_started.md#server-path) |
| Add Switchyard routing to a Rust application | Library Path | [`switchyard-libsy`](../crates/libsy/README.md) |
+| Add Switchyard routing to NeMo Relay | Native Plugin Path | [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md) |
The Server Path builds and runs the standalone `switchyard-server` binary.
@@ -27,3 +28,4 @@ The Server Path builds and runs the standalone `switchyard-server` binary.
- [`switchyard-libsy`](reference/rust_api.md#switchyard-libsy): embeddable routing algorithms
- [`switchyard-protocol`](reference/rust_api.md#switchyard-protocol): provider-neutral API types
- [`switchyard-translation`](../crates/switchyard-translation/README.md): protocol translation
+- [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md): native NeMo Relay integration
diff --git a/tests/relay_plugin/test_package_bundle.py b/tests/relay_plugin/test_package_bundle.py
new file mode 100644
index 000000000..05da044d8
--- /dev/null
+++ b/tests/relay_plugin/test_package_bundle.py
@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for the native Relay plugin bundle packager."""
+
+from __future__ import annotations
+
+import hashlib
+import subprocess
+import sys
+import tarfile
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
+CRATE_ROOT = REPOSITORY_ROOT / "crates" / "switchyard-nemo-relay-plugin"
+PACKAGER = CRATE_ROOT / "scripts" / "package_bundle.py"
+PACKAGE_NAME = "switchyard-nemo-relay-plugin"
+
+
+class PackageBundleTest(unittest.TestCase):
+ """Verify materialized and archived plugin bundle contents."""
+
+ def test_materializes_and_archives_supported_formats(self) -> None:
+ for archive_suffix in (".tar.gz", ".zip"):
+ with self.subTest(archive_suffix=archive_suffix), tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ library = root / "libswitchyard_nemo_relay_plugin.so"
+ library.write_bytes(b"compiled plugin")
+ output = root / "bundle"
+ archive = root / f"{PACKAGE_NAME}-0.2.0-linux-x86_64{archive_suffix}"
+
+ subprocess.run(
+ [
+ sys.executable,
+ str(PACKAGER),
+ "--library",
+ str(library),
+ "--output",
+ str(output),
+ "--archive",
+ str(archive),
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+
+ expected = {
+ "LICENSE",
+ "NOTICE",
+ "config.schema.json",
+ library.name,
+ "relay-plugin.toml",
+ }
+ self.assertEqual({path.name for path in output.iterdir()}, expected)
+ manifest = (output / "relay-plugin.toml").read_text(encoding="utf-8")
+ self.assertIn(f'artifact = "{library.name}"', manifest)
+ self.assertIn(hashlib.sha256(library.read_bytes()).hexdigest(), manifest)
+ self.assertNotIn("<platform-library-file>", manifest)
+ self.assertNotIn("<artifact-sha256>", manifest)
+ self.assertEqual(self.archive_members(archive), {f"{PACKAGE_NAME}/{name}" for name in expected})
+
+ def test_rejects_archive_inside_output_before_materializing_bundle(self) -> None:
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ library = root / "libswitchyard_nemo_relay_plugin.so"
+ library.write_bytes(b"compiled plugin")
+ output = root / "bundle"
+ archive = output / "plugin.zip"
+
+ result = subprocess.run(
+ [
+ sys.executable,
+ str(PACKAGER),
+ "--library",
+ str(library),
+ "--output",
+ str(output),
+ "--archive",
+ str(archive),
+ ],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("bundle archive must be outside output directory", result.stderr)
+ self.assertFalse(output.exists())
+ self.assertFalse(archive.exists())
+
+ @staticmethod
+ def archive_members(archive: Path) -> set[str]:
+ """Return regular-file paths from a supported bundle archive."""
+ if archive.name.endswith(".tar.gz"):
+ with tarfile.open(archive) as stream:
+ return {member.name for member in stream.getmembers() if member.isfile()}
+ with zipfile.ZipFile(archive) as stream:
+ return {member.filename for member in stream.infolist() if not member.is_dir()}
+
+
+if __name__ == "__main__":
+ unittest.main()