Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(relay): migrate dynamic plugin to `switchyard-runner` (#528)
Bryan Bednarski074b2bec3f16
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e9b7fa6dd..76884e283 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,16 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).### Added+- **NeMo Relay native plugin** — a dynamically loaded integration that loads+ Switchyard's standard TOML deployment and executes its `switchyard-runner`-+ supported configured routes in process. Managed calls require NeMo Relay+ `>=0.8.1,<0.9.0`; unknown models use Relay's continuation unchanged.++- **NeMo Relay routing marks** — routing-model usage, measured routing+ overhead, and selected-model decisions are emitted as ATOF marks. The final+ serving call remains represented only by Relay's outer LLM lifecycle event to+ prevent double-counting.+- **Advisor-gate routing** — new `advisor` route type pairing the servingexecutor with a stronger judge-only advisor that reviews terminal turns:APPROVE releases the buffered turn, REDO discards it and feeds the advisor'sdiff --git a/Cargo.lock b/Cargo.lockindex 89b712d9d..ab1f6c505 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -274,6 +274,9 @@ name = "bitflags"version = "2.13.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"+dependencies = [+ "serde_core",+][[package]]name = "borrow-or-share"@@ -334,6 +337,16 @@ dependencies = ["rand_core 0.10.1",]+[[package]]+name = "chrono"+version = "0.4.45"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"+dependencies = [+ "num-traits",+ "serde",+]+[[package]]name = "clap"version = "4.6.2"@@ -1226,6 +1239,34 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "nemo-relay-plugin"+version = "0.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ab4eaa56f021794a8b858c1ec6783679ad6798653e00a179166182bc4868bec"+dependencies = [+ "futures",+ "nemo-relay-types",+ "serde",+ "serde_json",+ "tokio",+ "tokio-util",+]++[[package]]+name = "nemo-relay-types"+version = "0.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1bd5df5a9286dd167f668939decc7c719c6b7e4253a2f758b5fd3b8af22cc3b0"+dependencies = [+ "bitflags",+ "chrono",+ "serde",+ "serde_json",+ "typed-builder",+ "uuid",+]+[[package]]name = "nu-ansi-term"version = "0.50.3"@@ -2325,6 +2366,25 @@ dependencies = ["wiremock",]+[[package]]+name = "switchyard-nemo-relay-plugin"+version = "0.2.0"+dependencies = [+ "futures-util",+ "http",+ "nemo-relay-plugin",+ "serde",+ "serde_json",+ "switchyard-llm-client",+ "switchyard-protocol",+ "switchyard-runner",+ "switchyard-translation",+ "tempfile",+ "tokio",+ "toml",+ "wiremock",+]+[[package]]name = "switchyard-protocol"version = "0.2.0"@@ -2644,6 +2704,7 @@ dependencies = ["bytes","futures-core","futures-sink",+ "futures-util","pin-project-lite","tokio",]@@ -2817,6 +2878,26 @@ version = "0.2.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"+[[package]]+name = "typed-builder"+version = "0.23.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda"+dependencies = [+ "typed-builder-macro",+]++[[package]]+name = "typed-builder-macro"+version = "0.23.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]+[[package]]name = "unicode-general-category"version = "1.1.0"@@ -2859,6 +2940,18 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"+[[package]]+name = "uuid"+version = "1.18.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"+dependencies = [+ "getrandom 0.3.4",+ "js-sys",+ "serde",+ "wasm-bindgen",+]+[[package]]name = "uuid-simd"version = "0.8.0"diff --git a/Cargo.toml b/Cargo.tomlindex 6cacbd29e..7a8fc60ef 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -9,6 +9,7 @@ members = ["crates/prefill-router","crates/switchyard-py","crates/protocol",+ "crates/switchyard-nemo-relay-plugin","crates/switchyard-runner","crates/switchyard-server","crates/switchyard-skill-distillation",@@ -34,6 +35,7 @@ http = "1"httpdate = "1"jsonschema = { version = "0.49.4", default-features = false }jsonptr = { version = "0.8.1", default-features = false, features = ["std", "json", "resolve"] }+nemo-relay-plugin = ">=0.8.1, <0.9.0"parking_lot = "0.12"rand = "0.10"regex = "1"diff --git a/README.md b/README.mdindex 265bb56b0..ded0df8f0 100644--- a/README.md+++ b/README.md@@ -21,6 +21,7 @@ algorithm you write yourself.- **Protocol Translation**: convert between OpenAI Chat, Anthropic Messages, and OpenAI Responses formats- **Multi-Backend Routing**: random routing, LLM-as-classifier routing, signal-driven stage-router, or your own algorithm- **Operational Metrics**: Prometheus metrics cover requests, errors, latency, tokens, and routing overhead+- **NeMo Relay Plugin**: run Switchyard-configured routes in NeMo Relay—including any routing algorithm supported by `switchyard-runner`—while Switchyard owns provider HTTP dispatch## Maturity@@ -129,6 +130,7 @@ configured LLM client selects one upstream format.- **[`switchyard-libsy`](crates/libsy/README.md)**: embed routing algorithms in a Rust application- **[`switchyard-protocol`](crates/protocol/README.md)**: provider-neutral request, response, and streaming types- **[`switchyard-translation`](crates/switchyard-translation/README.md)**: request, response, and stream translation+- **[`switchyard-nemo-relay-plugin`](crates/switchyard-nemo-relay-plugin/README.md)**: install Switchyard as a native NeMo Relay plugin## Communitydiff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.tomlnew file mode 100644index 000000000..ac9099f7f--- /dev/null+++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml@@ -0,0 +1,33 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++[package]+name = "switchyard-nemo-relay-plugin"+version.workspace = true+description = "Switchyard-owned HTTP routing plugin for NeMo Relay"+authors.workspace = true+edition.workspace = true+license.workspace = true+repository.workspace = true+rust-version.workspace = true+publish = false++[lib]+crate-type = ["cdylib"]++[dependencies]+futures-util.workspace = true+http.workspace = true+nemo-relay-plugin.workspace = true+serde.workspace = true+serde_json.workspace = true+switchyard-llm-client.workspace = true+switchyard-protocol.workspace = true+switchyard-runner.workspace = true+switchyard-translation.workspace = true+tokio.workspace = true+toml = "1.1"++[dev-dependencies]+tempfile = "3"+wiremock = "0.6"diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.mdnew file mode 100644index 000000000..d1359096a--- /dev/null+++ b/crates/switchyard-nemo-relay-plugin/README.md@@ -0,0 +1,118 @@+# Switchyard NeMo Relay Plugin++`switchyard-nemo-relay-plugin` is a native NeMo Relay dynamic plugin. It loads+a standard Switchyard TOML deployment from a file or Relay's nested plugin+configuration and executes its configured routes through `switchyard-runner`.++The plugin does not define a second routing or target configuration language.+`switchyard-server` and Relay therefore use the same targets, client pooling,+algorithm construction, retry policy, and route validation.++## Install++Build the platform bundle with the package script, then configure Relay to load+the generated `relay-plugin.toml` manifest. The plugin requires NeMo Relay+`>=0.8.1,<0.9.0`.++## Configure Relay++Use exactly one Switchyard deployment source. To share an existing deployment+file with `switchyard-server`, configure its path:++```toml+[[plugins.dynamic]]+manifest = "./plugins/switchyard/relay-plugin.toml"++[plugins.dynamic.config]+priority = 0+switchyard_config_path = "/etc/switchyard/routes.toml"+```++`switchyard_config_path` is a Switchyard version-1 TOML deployment, accepted by both+`switchyard-server` and `switchyard-runner`. See the+[server configuration guide](../switchyard-server/CONFIGURATION.md) for the+deployment schema and routing algorithms.++To keep the deployment in the Relay configuration, nest the same version-1+Switchyard configuration under `switchyard_config`:++```toml+[[plugins.dynamic]]+manifest = "./plugins/switchyard/relay-plugin.toml"++[plugins.dynamic.config]+priority = 0++[plugins.dynamic.config.switchyard_config]+schema_version = 1++[plugins.dynamic.config.switchyard_config.llm_clients.primary]+format = "openai_chat"+base_url = "https://example.test/v1"++[plugins.dynamic.config.switchyard_config.targets.default]+id = "example/model"+llm_client = "primary"++[plugins.dynamic.config.switchyard_config.routes.default]+id = "switchyard/default"+type = "passthrough"+target = "default"+```++## Request handling++For OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages calls,+the plugin decodes the Relay request and checks the requested model against the+deployment's route IDs.++- A configured route is executed by `switchyard-runner`.+- An unknown model calls Relay's continuation unchanged.+- The returned provider response is encoded back into the caller's wire format.+- Streaming responses are returned as unpolled translated streams; Relay owns+ cancellation and the outer serving-call lifecycle.++Each route's target client must use the caller's wire format: `openai_chat`,+`openai_responses`, or `anthropic_messages`. The runner selects the upstream+backend from that format rather than translating a route to a different+provider API. When one upstream model must serve multiple caller formats,+declare a target and route for each corresponding client format.++The plugin emits a routing request mark, routing-model call marks, measured+routing-overhead marks, and a selected-model decision mark. Token usage is+emitted as Switchyard metrics for both routing-model and answer-model calls;+Relay retains ownership of the outer LLM lifecycle.++## Observability++When Relay is configured with OTLP logs and metrics exporters, the plugin emits+typed telemetry through Relay's native plugin runtime:++- Routing request, decision, and overhead marks are Info logs.+- Per-routing-model call marks are Debug logs, including their outcome and+ latency, but not token usage.+- Terminal routing and response-finalization failures are Error logs. Their+ payload contains only the safe Switchyard failure summary; it excludes+ provider response bodies and free-form provider messages.+- Metrics use bounded attributes only: algorithm for+ `switchyard.routing.requests`; outcome for `switchyard.routing.llm_calls`+ and `switchyard.routing.llm_call.duration`;+ and safe failure kind, category, phase, and optional upstream HTTP status for+ `switchyard.routing.failures`. `switchyard.routing.overhead` records total+ routing latency, including routing-model calls; durations use milliseconds.+- `switchyard.routing.llm_tokens` records normalized token usage with+ `call_role` (`routing` or `answer`), configured `target_model`, and+ `token_type` attributes. A provider may omit usage for streaming responses;+ the plugin does not synthesize zero-value measurements.++The plugin does not attach sessions, requests, or provider messages as metric+attributes. `target_model` comes from the configured Switchyard target set,+rather than arbitrary caller input, keeping the metric cardinality bounded by+the deployment.++## Failure policy++`switchyard-llm-client` owns provider retry and route-candidate fallback+behavior. The plugin does not maintain a separate trusted-default target or+rerun routing after an execution failure. Failures outside the shared runner,+including response translation failures, are returned to Relay.diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.jsonnew file mode 100644index 000000000..61f178a12--- /dev/null+++ b/crates/switchyard-nemo-relay-plugin/config.schema.json@@ -0,0 +1,40 @@+{+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "title": "Switchyard NeMo Relay Plugin",+ "description": "Runs a Switchyard deployment through its shared in-process runner.",+ "type": "object",+ "additionalProperties": false,+ "oneOf": [+ { "required": ["switchyard_config_path"] },+ { "required": ["switchyard_config"] }+ ],+ "properties": {+ "priority": {+ "type": "integer",+ "default": 0,+ "description": "NeMo Relay execution-intercept priority."+ },+ "executor": {+ "type": "object",+ "additionalProperties": false,+ "description": "Optional NeMo Relay SDK executor override.",+ "properties": {+ "worker_threads": {+ "type": "integer",+ "minimum": 1,+ "description": "Number of worker threads for the plugin's SDK-owned executor."+ }+ }+ },+ "switchyard_config_path": {+ "type": "string",+ "minLength": 1,+ "description": "Path to a Switchyard version-1 TOML deployment shared with switchyard-server."+ },+ "switchyard_config": {+ "type": "object",+ "minProperties": 1,+ "description": "Inline Switchyard version-1 deployment using the same schema as switchyard-server."+ }+ }+}diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.tomlnew file mode 100644index 000000000..e2f23aa17--- /dev/null+++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml@@ -0,0 +1,31 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++manifest_version = 1++[plugin]+id = "nvidia.switchyard"+kind = "rust_dynamic"++[compat]+relay = ">=0.8.1,<0.9.0"+native_api = "1"++[defaults]+enabled = false++[capabilities]+items = ["plugin_native", "config_schema"]++[config_schema]+path = "config.schema.json"++[source]+artifact = "<platform-library-file>"++[integrity]+sha256 = "sha256:<artifact-sha256>"++[load]+library = "<platform-library-file>"+symbol = "nemo_relay_register_plugin"diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.pynew file mode 100644index 000000000..3b79af769--- /dev/null+++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py@@ -0,0 +1,104 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Materialize the minimal Relay plugin bundle from a compiled cdylib."""++from __future__ import annotations++import argparse+import hashlib+import shutil+import tarfile+import zipfile+from pathlib import Path++CRATE_ROOT = Path(__file__).resolve().parents[1]+REPOSITORY_ROOT = CRATE_ROOT.parents[1]+PACKAGE_NAME = "switchyard-nemo-relay-plugin"+++def digest(path: Path) -> str:+ """Return the lowercase SHA-256 digest for a file."""+ value = hashlib.sha256()+ with path.open("rb") as stream:+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):+ value.update(chunk)+ return value.hexdigest()+++def archive_bundle(bundle: Path, archive: Path) -> None:+ """Archive a materialized bundle under the stable package directory name."""+ if archive.parent.is_relative_to(bundle):+ raise ValueError(f"bundle archive must be outside output directory: {archive}")+ archive.parent.mkdir(parents=True, exist_ok=True)+ if archive.exists():+ raise ValueError(f"bundle archive already exists: {archive}")++ if archive.name.endswith(".tar.gz"):+ with tarfile.open(archive, "w:gz") as stream:+ stream.add(bundle, arcname=PACKAGE_NAME)+ return++ if archive.suffix == ".zip":+ with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as stream:+ for path in sorted(bundle.rglob("*")):+ if path.is_file():+ stream.write(path, Path(PACKAGE_NAME) / path.relative_to(bundle))+ return++ raise ValueError("bundle archive must end in .tar.gz or .zip")+++def main() -> None:+ """Materialize a Relay-loadable plugin bundle in an empty directory."""+ parser = argparse.ArgumentParser()+ parser.add_argument("--library", required=True, type=Path)+ parser.add_argument("--output", required=True, type=Path)+ parser.add_argument("--archive", type=Path)+ args = parser.parse_args()++ library = args.library.resolve()+ if not library.is_file():+ parser.error(f"compiled plugin library does not exist: {library}")++ manifest = (CRATE_ROOT / "relay-plugin.toml").read_text(encoding="utf-8")+ placeholders = ("<platform-library-file>", "<artifact-sha256>")+ missing = [placeholder for placeholder in placeholders if placeholder not in manifest]+ if missing:+ parser.error(f"plugin manifest is missing placeholders: {', '.join(missing)}")++ output = args.output.resolve()+ if output.exists() and not output.is_dir():+ parser.error(f"bundle output exists and is not a directory: {output}")+ if output.is_dir() and any(output.iterdir()):+ parser.error(f"bundle output directory must be empty: {output}")++ archive = args.archive.resolve() if args.archive is not None else None+ if archive is not None and archive.parent.is_relative_to(output):+ parser.error(f"bundle archive must be outside output directory: {archive}")+ output.mkdir(parents=True, exist_ok=True)++ artifact = output / library.name+ shutil.copy2(library, artifact)+ shutil.copy2(CRATE_ROOT / "config.schema.json", output / "config.schema.json")+ for filename in ("LICENSE", "NOTICE"):+ shutil.copy2(REPOSITORY_ROOT / filename, output / filename)++ artifact_digest = digest(artifact)+ manifest = manifest.replace("<platform-library-file>", artifact.name)+ manifest = manifest.replace("<artifact-sha256>", artifact_digest)+ (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8")++ if archive is None:+ print(output)+ return++ try:+ archive_bundle(output, archive)+ except ValueError as error:+ parser.error(str(error))+ print(archive)+++if __name__ == "__main__":+ main()diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rsnew file mode 100644index 000000000..2df6386b3--- /dev/null+++ b/crates/switchyard-nemo-relay-plugin/src/config.rs@@ -0,0 +1,152 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use std::path::PathBuf;++use serde::Deserialize;+use serde_json::{Map, Value};+use switchyard_protocol::WireFormat;+use switchyard_runner::Runner;++pub(crate) fn protocol_from_call(name: &str) -> Option<WireFormat> {+ match name {+ "openai.chat_completions" => Some(WireFormat::OpenAiChat),+ "openai.responses" => Some(WireFormat::OpenAiResponses),+ "anthropic.messages" => Some(WireFormat::AnthropicMessages),+ _ => None,+ }+}++#[derive(Debug, Deserialize)]+#[serde(deny_unknown_fields)]+pub(crate) struct SwitchyardConfig {+ #[serde(default)]+ pub(crate) priority: i32,+ #[serde(default)]+ pub(crate) switchyard_config_path: Option<PathBuf>,+ #[serde(default)]+ pub(crate) switchyard_config: Option<Map<String, Value>>,+}++impl SwitchyardConfig {+ pub(crate) fn load_runner(&self) -> Result<Runner, String> {+ match (&self.switchyard_config_path, &self.switchyard_config) {+ (Some(path), None) => Runner::load(path).map_err(|error| error.to_string()),+ (None, Some(config)) => toml::to_string(config)+ .map_err(|error| format!("failed to serialize Switchyard configuration: {error}"))+ .and_then(|source| Runner::from_toml(&source).map_err(|error| error.to_string())),+ (Some(_), Some(_)) => Err(+ "configure exactly one of switchyard_config_path or switchyard_config".to_string(),+ ),+ (None, None) => {+ Err("configure one of switchyard_config_path or switchyard_config".to_string())+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use std::io::Write;++ use serde_json::json;+ use tempfile::NamedTempFile;++ use super::*;++ fn inline_deployment() -> Map<String, Value> {+ json!({+ "schema_version": 1,+ "llm_clients": {+ "primary": {+ "format": "openai_chat",+ "base_url": "https://example.test/v1"+ }+ },+ "targets": {+ "default": {+ "id": "example/model",+ "llm_client": "primary"+ }+ },+ "routes": {+ "default": {+ "id": "switchyard/default",+ "type": "passthrough",+ "target": "default"+ }+ }+ })+ .as_object()+ .unwrap()+ .clone()+ }++ #[test]+ fn maps_supported_relay_call_names_to_wire_formats() {+ let cases = [+ ("openai.chat_completions", Some(WireFormat::OpenAiChat)),+ ("openai.responses", Some(WireFormat::OpenAiResponses)),+ ("anthropic.messages", Some(WireFormat::AnthropicMessages)),+ ("unsupported.call", None),+ ];++ for (name, expected) in cases {+ assert_eq!(protocol_from_call(name), expected, "{name}");+ }+ }++ #[test]+ fn builds_runner_from_inline_deployment() {+ let config = SwitchyardConfig {+ priority: 0,+ switchyard_config_path: None,+ switchyard_config: Some(inline_deployment()),+ };++ let runner = config.load_runner().unwrap();+ assert!(runner.route("switchyard/default").is_some());+ }++ #[test]+ fn builds_runner_from_deployment_path() {+ let mut deployment = NamedTempFile::new().unwrap();+ write!(+ deployment,+ "{}",+ toml::to_string(&inline_deployment()).unwrap()+ )+ .unwrap();+ let config = SwitchyardConfig {+ priority: 0,+ switchyard_config_path: Some(deployment.path().to_path_buf()),+ switchyard_config: None,+ };++ let runner = config.load_runner().unwrap();+ assert!(runner.route("switchyard/default").is_some());+ }++ #[test]+ fn requires_exactly_one_configuration_source() {+ let neither = SwitchyardConfig {+ priority: 0,+ switchyard_config_path: None,+ switchyard_config: None,+ };+ assert!(matches!(+ neither.load_runner(),+ Err(error) if error.contains("configure one of switchyard_config_path or switchyard_config")+ ));++ let both = SwitchyardConfig {+ priority: 0,+ switchyard_config_path: Some(PathBuf::from("routes.toml")),+ switchyard_config: Some(inline_deployment()),+ };+ assert!(matches!(+ both.load_runner(),+ Err(error) if error.contains("configure exactly one of switchyard_config_path or switchyard_config")+ ));+ }+}diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rsnew file mode 100644index 000000000..853a2e81b--- /dev/null+++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs@@ -0,0 +1,173 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++mod config;+mod runtime;+mod translation;++use std::sync::Arc;++use nemo_relay_plugin::{+ ConfigDiagnostic, DiagnosticLevel, Json, NativePlugin, PluginContext, PluginRuntime,+};+use serde_json::Map;++use crate::config::{SwitchyardConfig, protocol_from_call};+use crate::runtime::{SwitchyardRuntime, emit_event, emit_events};++#[derive(Default)]+struct SwitchyardPlugin;++impl NativePlugin for SwitchyardPlugin {+ fn plugin_kind(&self) -> &str {+ "nvidia.switchyard"+ }++ fn allows_multiple_components(&self) -> bool {+ false+ }++ fn validate(&self, plugin_config: &Map<String, Json>) -> Vec<ConfigDiagnostic> {+ // Allow `SwitchyardRuntime::new` to run for accurate validation diagnostics.+ match parse_config(plugin_config).and_then(SwitchyardRuntime::new) {+ Ok(_) => Vec::new(),+ Err(message) => vec![ConfigDiagnostic {+ level: DiagnosticLevel::Error,+ code: "switchyard.invalid_config".into(),+ component: Some("nvidia.switchyard".into()),+ field: Some("config".into()),+ message,+ }],+ }+ }++ fn register(+ &mut self,+ plugin_config: &Map<String, Json>,+ ctx: &mut PluginContext<'_>,+ ) -> nemo_relay_plugin::Result<()> {+ let config = parse_config(plugin_config)?;+ let priority = config.priority;+ let runtime = Arc::new(SwitchyardRuntime::new(config)?);+ let plugin_runtime = ctx.runtime();+ register_buffered(ctx, priority, Arc::clone(&runtime), plugin_runtime.clone())?;+ register_stream(ctx, priority, runtime, plugin_runtime)?;+ Ok(())+ }+}++fn register_buffered(+ ctx: &mut PluginContext<'_>,+ priority: i32,+ runtime: Arc<SwitchyardRuntime>,+ plugin_runtime: PluginRuntime,+) -> Result<(), String> {+ ctx.register_llm_execution_intercept(+ "switchyard.runner.buffered",+ priority,+ move |name, request, next| {+ let runtime = Arc::clone(&runtime);+ let plugin_runtime = plugin_runtime.clone();+ async move {+ let Some(inbound) = protocol_from_call(&name) else {+ return next.call(request).await;+ };+ if !request_model(&request).is_some_and(|model| runtime.manages_model(model)) {+ return next.call(request).await;+ }+ let decoded = runtime.decode_request(inbound, request, false)?;+ let execution = runtime.execute_buffered(inbound, decoded).await;+ emit_events(&plugin_runtime, execution.events);+ execution.result+ }+ },+ )+}++fn register_stream(+ ctx: &mut PluginContext<'_>,+ priority: i32,+ runtime: Arc<SwitchyardRuntime>,+ plugin_runtime: PluginRuntime,+) -> Result<(), String> {+ ctx.register_llm_stream_execution_intercept(+ "switchyard.runner.streaming",+ priority,+ move |name, request, next| {+ let runtime = Arc::clone(&runtime);+ let plugin_runtime = plugin_runtime.clone();+ async move {+ let Some(inbound) = protocol_from_call(&name) else {+ return next.call(request).await;+ };+ if !request_model(&request).is_some_and(|model| runtime.manages_model(model)) {+ return next.call(request).await;+ }+ let decoded = runtime.decode_request(inbound, request, true)?;+ let stream_plugin_runtime = plugin_runtime.clone();+ let execution = runtime+ .execute_stream(+ inbound,+ decoded,+ Arc::new(move |event| emit_event(&stream_plugin_runtime, event)),+ )+ .await;+ emit_events(&plugin_runtime, execution.events);+ execution.result+ }+ },+ )+}++fn parse_config(plugin_config: &Map<String, Json>) -> Result<SwitchyardConfig, String> {+ let mut config = plugin_config.clone();+ config.remove("executor");+ serde_json::from_value(Json::Object(config))+ .map_err(|error| format!("invalid Switchyard configuration: {error}"))+}++fn request_model(request: &nemo_relay_plugin::LlmRequest) -> Option<&str> {+ request.content.get("model").and_then(Json::as_str)+}++nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default);++#[cfg(test)]+mod tests {+ use serde_json::json;++ use super::*;++ #[test]+ fn plugin_configuration_requires_a_switchyard_source() {+ let config = json!({"priority": 0});+ assert!(matches!(+ parse_config(config.as_object().unwrap()).and_then(SwitchyardRuntime::new),+ Err(error) if error.contains("switchyard_config_path or switchyard_config")+ ));+ }++ #[test]+ fn plugin_configuration_ignores_the_sdk_executor_override() {+ let config = json!({+ "executor": {"worker_threads": 4},+ "switchyard_config_path": "routes.toml"+ });+ assert!(parse_config(config.as_object().unwrap()).is_ok());+ }++ #[test]+ fn request_model_reads_only_a_top_level_string() {+ let request = nemo_relay_plugin::LlmRequest {+ headers: Map::new(),+ content: json!({"model": "switchyard/default", "messages": []}),+ };+ assert_eq!(request_model(&request), Some("switchyard/default"));++ let request = nemo_relay_plugin::LlmRequest {+ headers: Map::new(),+ content: json!({"model": {"name": "switchyard/default"}}),+ };+ assert_eq!(request_model(&request), None);+ }+}diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rsnew file mode 100644index 000000000..7b5a12fb9--- /dev/null+++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs@@ -0,0 +1,1412 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use std::pin::Pin;+use std::sync::{Arc, Mutex};++use futures_util::{Stream, StreamExt};+use http::StatusCode;+use nemo_relay_plugin::{+ Json, LlmRequest as RelayRequest, LogSeverity, MetricKind, MetricMeasurement, MetricValueType,+ PluginRuntime,+};+use serde_json::{Map, json};+use switchyard_llm_client::{LlmCallObservation, RunObservation, RunObserver};+use switchyard_protocol::{+ LlmClientError, LlmResponse, LlmResponseChunk, LlmStreamError, Metadata, ProviderExtensions,+ Request, Response, Usage, WireFormat,+};+use switchyard_runner::{Route, RouteErrorSummary, Runner, stream_error_summary};+use switchyard_translation::{TranslationEngine, encode_stream_with_extensions};++use crate::config::SwitchyardConfig;+use crate::translation;++#[derive(Debug)]+pub(crate) struct RoutingMark {+ pub(crate) name: String,+ pub(crate) data: Json,+ pub(crate) metadata: Json,+ pub(crate) severity: Option<LogSeverity>,+}++#[derive(Debug)]+pub(crate) struct RoutingMetric {+ pub(crate) name: String,+ pub(crate) measurements: Vec<MetricMeasurement>,+ pub(crate) metadata: Json,+}++#[derive(Debug)]+pub(crate) enum RoutingEvent {+ Mark(RoutingMark),+ Metric(RoutingMetric),+}++struct MetricDescriptor<'a> {+ name: &'a str,+ kind: MetricKind,+ value_type: MetricValueType,+ unit: Option<&'a str>,+ description: &'a str,+}++pub(crate) type ReturnedEventStream = Pin<Box<dyn Stream<Item = Result<Json, String>> + Send>>;+pub(crate) type RoutingEventEmitter = Arc<dyn Fn(RoutingEvent) + Send + Sync>;++pub(crate) struct Execution<T> {+ pub(crate) result: Result<T, String>,+ pub(crate) events: Vec<RoutingEvent>,+}++pub(crate) struct SwitchyardRuntime {+ runner: Runner,+ translation: TranslationEngine,+}++impl SwitchyardRuntime {+ pub(crate) fn new(config: SwitchyardConfig) -> Result<Self, String> {+ Ok(Self {+ runner: config.load_runner()?,+ translation: TranslationEngine::default(),+ })+ }++ pub(crate) fn manages_model(&self, model: &str) -> bool {+ self.runner.route(model).is_some()+ }++ pub(crate) fn decode_request(+ &self,+ inbound: WireFormat,+ request: RelayRequest,+ streaming: bool,+ ) -> Result<Request, String> {+ let mut llm_request = translation::decode_request(&self.translation, inbound, &request)?;+ llm_request.stream = streaming;+ let headers = string_headers(&request.headers);+ let mut metadata = Metadata::from_headers(&headers);+ let relay_gateway_placeholder = !headers.contains_key("x-switchyard-session-id")+ && headers+ .get("x-nemo-relay-source")+ .and_then(|value| value.to_str().ok())+ == Some("gateway")+ && metadata.session_id.as_deref() == Some("gateway-gateway");+ if relay_gateway_placeholder {+ metadata.session_id = None;+ }+ metadata.http_headers = Some(headers);+ Ok(Request {+ llm_request,+ raw_request: Some(request.content),+ metadata: Some(metadata),+ })+ }++ pub(crate) async fn execute_buffered(+ &self,+ inbound: WireFormat,+ request: Request,+ ) -> Execution<Json> {+ let request_extensions = request.llm_request.extensions.clone();+ let Execution { result, mut events } = self.execute(inbound, request).await;+ let (result, finalization_failed) = match result {+ Ok(response) => {+ let result = finalize_buffered_response(+ &self.translation,+ inbound,+ response,+ &request_extensions,+ );+ let failed = result.is_err();+ (result, failed)+ }+ Err(error) => (Err(error), false),+ };+ if finalization_failed {+ self.error_mark(&mut events, "response_finalization", None);+ }+ Execution { result, events }+ }++ pub(crate) async fn execute_stream(+ &self,+ inbound: WireFormat,+ request: Request,+ emit_event: RoutingEventEmitter,+ ) -> Execution<ReturnedEventStream> {+ let request_extensions = request.llm_request.extensions.clone();+ let Execution { result, mut events } = self.execute(inbound, request).await;+ let (result, finalization_failed) = match result {+ Ok(response) => {+ let metadata = events+ .iter()+ .find_map(|event| match event {+ RoutingEvent::Mark(mark) => Some(mark.metadata.clone()),+ RoutingEvent::Metric(_) => None,+ })+ .unwrap_or_else(|| Json::Object(Map::new()));+ let result =+ returned_events(response, inbound, &request_extensions, metadata, emit_event);+ let failed = result.is_err();+ (result, failed)+ }+ Err(error) => (Err(error), false),+ };+ if finalization_failed {+ self.error_mark(&mut events, "response_finalization", None);+ }+ Execution { result, events }+ }++ async fn execute(&self, inbound: WireFormat, request: Request) -> Execution<Response> {+ let Some(route) = self.route(&request) else {+ return Execution {+ result: Err("Switchyard has no route for this request model".into()),+ events: Vec::new(),+ };+ };+ let metadata = identity_metadata(request.metadata.as_ref());+ let mut events = vec![RoutingEvent::Mark(RoutingMark {+ name: "switchyard.routing.requested".into(),+ data: json!({"algorithm": route.algorithm_name()}),+ metadata: metadata.clone(),+ severity: Some(LogSeverity::Info),+ })];+ events.push(request_metric(route.algorithm_name(), metadata.clone()));+ if let Err(error) = route.check_caller_format(inbound) {+ self.error_mark(&mut events, "caller_format", None);+ return Execution {+ result: Err(format!("Switchyard caller format is incompatible: {error}")),+ events,+ };+ }+ let observations = Arc::new(Mutex::new(Vec::new()));+ let observed = Arc::clone(&observations);+ let observer: RunObserver = Arc::new(move |observation| {+ observed+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .push(observation);+ });+ match route.execute(request, Some(observer)).await {+ Ok(output) => {+ self.emit_observations(&mut events, take_observations(&observations), &metadata);+ events.push(RoutingEvent::Mark(RoutingMark {+ name: "switchyard.routing.decision".into(),+ data: json!({+ "algorithm": route.algorithm_name(),+ "selected_model": output.selected_model,+ }),+ metadata,+ severity: Some(LogSeverity::Info),+ }));+ Execution {+ result: Ok(output.response),+ events,+ }+ }+ Err(error) => {+ self.emit_observations(&mut events, take_observations(&observations), &metadata);+ self.route_execution_error_mark(+ &mut events,+ &error.execution_error_summary(),+ None,+ );+ Execution {+ result: Err("Switchyard route execution failed".into()),+ events,+ }+ }+ }+ }++ fn route(&self, request: &Request) -> Option<&Route> {+ request+ .llm_request+ .model+ .as_deref()+ .and_then(|model| self.runner.route(model))+ }++ fn emit_observations(+ &self,+ events: &mut Vec<RoutingEvent>,+ observations: Vec<RunObservation>,+ metadata: &Json,+ ) {+ let mut call_index = 0;+ for observation in observations {+ match observation {+ RunObservation::LlmCall(call) => {+ call_index += 1;+ self.routing_call_events(events, call, call_index, metadata);+ }+ RunObservation::RoutingOverhead(duration) => {+ let latency_ms = duration.as_secs_f64() * 1_000.0;+ events.push(RoutingEvent::Mark(RoutingMark {+ name: "switchyard.routing.overhead".into(),+ data: json!({"latency_ms": latency_ms}),+ metadata: metadata.clone(),+ severity: Some(LogSeverity::Info),+ }));+ events.push(routing_overhead_metric(latency_ms, metadata.clone()));+ }+ RunObservation::AnswerCall(call) => {+ events.extend(token_usage_metrics("answer", &call, metadata));+ }+ }+ }+ }++ fn routing_call_events(+ &self,+ events: &mut Vec<RoutingEvent>,+ call: LlmCallObservation,+ call_index: usize,+ metadata: &Json,+ ) {+ let outcome = if call.is_success { "ok" } else { "error" };+ let latency_ms = call.duration.as_secs_f64() * 1_000.0;+ let token_metrics = token_usage_metrics("routing", &call, metadata);+ events.push(RoutingEvent::Mark(RoutingMark {+ name: "switchyard.routing.llm_call".into(),+ data: json!({+ "call_index": call_index,+ "selected_model": call.selected_model.as_str(),+ "call_role": "routing",+ "outcome": outcome,+ "latency_ms": latency_ms,+ }),+ metadata: metadata.clone(),+ severity: Some(LogSeverity::Debug),+ }));+ events.extend(routing_call_metrics(outcome, latency_ms, metadata.clone()));+ events.extend(token_metrics);+ }++ fn error_mark(+ &self,+ events: &mut Vec<RoutingEvent>,+ failure_kind: &str,+ metadata: Option<&Json>,+ ) {+ let metadata = metadata+ .cloned()+ .unwrap_or_else(|| event_metadata(events).unwrap_or_else(|| Json::Object(Map::new())));+ events.push(RoutingEvent::Mark(RoutingMark {+ name: "switchyard.routing.error".into(),+ data: json!({"failure_kind": failure_kind}),+ metadata: metadata.clone(),+ severity: Some(LogSeverity::Error),+ }));+ events.push(failure_metric(failure_kind, None, None, None, metadata));+ }++ fn route_execution_error_mark(+ &self,+ events: &mut Vec<RoutingEvent>,+ summary: &RouteErrorSummary,+ metadata: Option<&Json>,+ ) {+ let metadata = metadata+ .cloned()+ .unwrap_or_else(|| event_metadata(events).unwrap_or_else(|| Json::Object(Map::new())));+ events.extend(route_execution_error_events(summary, metadata));+ }+}++pub(crate) fn emit_events(runtime: &PluginRuntime, events: Vec<RoutingEvent>) {+ for event in events {+ emit_event(runtime, event);+ }+}++pub(crate) fn emit_event(runtime: &PluginRuntime, event: RoutingEvent) {+ let result = match event {+ RoutingEvent::Mark(mark) => runtime+ .emit_mark_with_options(+ &mark.name,+ Some(&mark.data),+ Some(&mark.metadata),+ None,+ mark.severity,+ )+ .map_err(|error| ("routing mark", mark.name, error)),+ RoutingEvent::Metric(metric) => runtime+ .emit_metric(&metric.name, metric.measurements, Some(&metric.metadata))+ .map_err(|error| ("routing metric", metric.name, error)),+ };+ if let Err((kind, name, error)) = result {+ eprintln!("Switchyard could not emit {kind} {name:?}: {error}");+ }+}++fn take_observations(observations: &Mutex<Vec<RunObservation>>) -> Vec<RunObservation> {+ std::mem::take(+ &mut *observations+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner()),+ )+}++fn finalize_buffered_response(+ translation_engine: &TranslationEngine,+ inbound: WireFormat,+ response: Response,+ request_extensions: &ProviderExtensions,+) -> Result<Json, String> {+ let LlmResponse::Agg(response) = response.llm_response else {+ return Err("Switchyard returned a stream for a buffered request".into());+ };+ translation::encode_response(translation_engine, inbound, &response, request_extensions)+}++fn returned_events(+ response: Response,+ inbound: WireFormat,+ request_extensions: &ProviderExtensions,+ metadata: Json,+ emit_event: RoutingEventEmitter,+) -> Result<ReturnedEventStream, String> {+ let served_model = response.served_model().cloned();+ let (chunks, observe_stream_usage) = match response.llm_response {+ LlmResponse::Agg(response) => (response.into_stream(), false),+ LlmResponse::Stream(chunks) => (chunks, true),+ };+ let mut latest_usage = None;+ let mut terminal_seen = false;+ let mut usage_emitted = false;+ let mut stream_failed = false;+ let mut failure_emitted = false;+ let chunks = Box::pin(chunks.map(move |item| {+ let in_band_error = item+ .as_ref()+ .ok()+ .and_then(|event| normalized_stream_error(event.normalized()));+ let stream_error = in_band_error.as_ref().or_else(|| item.as_ref().err());+ stream_failed |= stream_error.is_some();+ if let Some(error) = stream_error+ && !failure_emitted+ {+ for event in route_execution_error_events(+ &stream_error_summary(error, served_model.as_ref()),+ metadata.clone(),+ ) {+ emit_event(event);+ }+ failure_emitted = true;+ }+ if observe_stream_usage && !stream_failed && !usage_emitted {+ if let Ok(event) = &item {+ for chunk in event.normalized() {+ match chunk {+ LlmResponseChunk::Usage(usage) => latest_usage = Some(usage.clone()),+ LlmResponseChunk::MessageStop { .. } => terminal_seen = true,+ _ => {}+ }+ }+ }+ if terminal_seen+ && let (Some(model), Some(usage)) = (served_model.as_ref(), latest_usage.as_ref())+ {+ for event in usage_metrics("answer", model.as_str(), usage, &metadata) {+ emit_event(event);+ }+ usage_emitted = true;+ }+ }+ item+ }));+ let events = encode_stream_with_extensions(chunks, inbound, None, request_extensions)+ .map_err(|error| format!("Switchyard response stream setup failed: {error}"))?;+ Ok(Box::pin(+ events.map(|item| item.map_err(relay_stream_error)),+ ))+}++/// Produces a telemetry-only error summary source for a normalized in-band failure.+///+/// The translation layer retains the original provider error event for client delivery;+/// this conversion is only used to classify the failure without putting its message in marks.+fn normalized_stream_error(chunks: &[LlmResponseChunk]) -> Option<LlmClientError> {+ chunks.iter().find_map(|chunk| match chunk {+ LlmResponseChunk::DecodeError { message } => {+ Some(LlmClientError::ResponseTranslation(message.clone()))+ }+ LlmResponseChunk::StreamError { message } => Some(LlmClientError::UpstreamHttp {+ status: StatusCode::BAD_GATEWAY,+ body: message.clone(),+ }),+ _ => None,+ })+}++/// Converts translation errors to Relay's string-only stream-error boundary.+///+/// `LlmStreamError::Upstream` retains a provider-shaped JSON value so its caller can+/// make protocol decisions, but neither that value nor a client error's source message+/// is safe for Relay's outer observability scope. Preserve the error result while+/// exposing only a stable failure class.+fn relay_stream_error(error: LlmStreamError) -> String {+ match error {+ LlmStreamError::Upstream(_) => "Switchyard upstream stream error".into(),+ LlmStreamError::Client(error) => format!(+ "Switchyard stream error: {}",+ stream_error_summary(&error, None).kind.as_str()+ ),+ }+}++fn route_execution_error_mark(summary: &RouteErrorSummary, metadata: Json) -> RoutingMark {+ RoutingMark {+ name: "switchyard.routing.error".into(),+ data: json!({+ "failure_kind": "route_execution",+ "category": summary.kind.as_str(),+ "phase": summary.phase.as_str(),+ "upstream_status": summary.upstream_status,+ "target": summary.target.as_ref().map(|target| target.as_str()),+ }),+ metadata,+ severity: Some(LogSeverity::Error),+ }+}++fn route_execution_error_events(summary: &RouteErrorSummary, metadata: Json) -> Vec<RoutingEvent> {+ vec![+ RoutingEvent::Mark(route_execution_error_mark(summary, metadata.clone())),+ failure_metric(+ "route_execution",+ Some(summary.kind.as_str()),+ Some(summary.phase.as_str()),+ summary.upstream_status,+ metadata,+ ),+ ]+}++fn event_metadata(events: &[RoutingEvent]) -> Option<Json> {+ events.iter().find_map(|event| match event {+ RoutingEvent::Mark(mark) => Some(mark.metadata.clone()),+ RoutingEvent::Metric(_) => None,+ })+}++fn request_metric(algorithm: &str, metadata: Json) -> RoutingEvent {+ counter_metric(+ "switchyard.routing.requests",+ "Requests managed by Switchyard routing.",+ json!({"algorithm": algorithm}),+ metadata,+ )+}++fn routing_call_metrics(outcome: &str, latency_ms: f64, metadata: Json) -> [RoutingEvent; 2] {+ let attributes = json!({"outcome": outcome});+ [+ counter_metric(+ "switchyard.routing.llm_calls",+ "Switchyard model calls made while routing.",+ attributes.clone(),+ metadata.clone(),+ ),+ histogram_metric(+ "switchyard.routing.llm_call.duration",+ "Duration of Switchyard model calls made while routing.",+ latency_ms,+ attributes,+ metadata,+ ),+ ]+}++fn routing_overhead_metric(latency_ms: f64, metadata: Json) -> RoutingEvent {+ histogram_metric(+ "switchyard.routing.overhead",+ "Time needed to produce the Switchyard routing outcome, including routing model calls.",+ latency_ms,+ json!({}),+ metadata,+ )+}++fn token_usage_metrics(+ call_role: &str,+ call: &LlmCallObservation,+ metadata: &Json,+) -> Vec<RoutingEvent> {+ let Some(usage) = call.usage.as_ref() else {+ return Vec::new();+ };+ usage_metrics(call_role, call.selected_model.as_str(), usage, metadata)+}++fn usage_metrics(+ call_role: &str,+ target_model: &str,+ usage: &Usage,+ metadata: &Json,+) -> Vec<RoutingEvent> {+ [+ ("input", usage.input_tokens),+ ("cached_input", usage.cached_input_tokens()),+ ("cache_creation_input", usage.cache_creation_input_tokens()),+ ("output", usage.output_tokens),+ ("reasoning", usage.reasoning_tokens),+ ("total", usage.total_tokens),+ ]+ .into_iter()+ .filter_map(|(token_type, value)| {+ value.map(|value| {+ metric(+ MetricDescriptor {+ name: "switchyard.routing.llm_tokens",+ kind: MetricKind::Counter,+ value_type: MetricValueType::U64,+ unit: Some("{token}"),+ description: "Normalized tokens used by Switchyard model calls.",+ },+ json!(value),+ json!({+ "call_role": call_role,+ "target_model": target_model,+ "token_type": token_type,+ }),+ metadata.clone(),+ )+ })+ })+ .collect()+}++fn failure_metric(+ failure_kind: &str,+ category: Option<&str>,+ phase: Option<&str>,+ upstream_status: Option<u16>,+ metadata: Json,+) -> RoutingEvent {+ let mut attributes = Map::new();+ attributes.insert("failure_kind".into(), Json::String(failure_kind.into()));+ if let Some(category) = category {+ attributes.insert("category".into(), Json::String(category.into()));+ }+ if let Some(phase) = phase {+ attributes.insert("phase".into(), Json::String(phase.into()));+ }+ if let Some(upstream_status) = upstream_status {+ attributes.insert("upstream_status".into(), Json::from(upstream_status));+ }+ counter_metric(+ "switchyard.routing.failures",+ "Terminal Switchyard routing failures.",+ Json::Object(attributes),+ metadata,+ )+}++fn counter_metric(name: &str, description: &str, attributes: Json, metadata: Json) -> RoutingEvent {+ metric(+ MetricDescriptor {+ name,+ kind: MetricKind::Counter,+ value_type: MetricValueType::U64,+ unit: Some("{event}"),+ description,+ },+ json!(1),+ attributes,+ metadata,+ )+}++fn histogram_metric(+ name: &str,+ description: &str,+ value: f64,+ attributes: Json,+ metadata: Json,+) -> RoutingEvent {+ metric(+ MetricDescriptor {+ name,+ kind: MetricKind::Histogram,+ value_type: MetricValueType::F64,+ unit: Some("ms"),+ description,+ },+ json!(value),+ attributes,+ metadata,+ )+}++fn metric(+ descriptor: MetricDescriptor<'_>,+ value: Json,+ attributes: Json,+ metadata: Json,+) -> RoutingEvent {+ RoutingEvent::Metric(RoutingMetric {+ name: descriptor.name.into(),+ measurements: vec![MetricMeasurement {+ name: descriptor.name.into(),+ kind: descriptor.kind,+ value_type: descriptor.value_type,+ value,+ unit: descriptor.unit.map(Into::into),+ description: Some(descriptor.description.into()),+ attributes: Some(attributes),+ boundaries: None,+ }],+ metadata,+ })+}++fn string_headers(headers: &Map<String, Json>) -> http::HeaderMap {+ let mut parsed = http::HeaderMap::with_capacity(headers.len());+ for (name, value) in headers {+ let Some(value) = value.as_str() else {+ continue;+ };+ let (Ok(name), Ok(value)) = (+ http::HeaderName::from_bytes(name.as_bytes()),+ http::HeaderValue::from_str(value),+ ) else {+ continue;+ };+ parsed.insert(name, value);+ }+ parsed+}++fn identity_metadata(metadata: Option<&Metadata>) -> Json {+ json!({+ "session_id": metadata.and_then(|value| value.session_id.as_deref()),+ "agent_id": metadata.and_then(|value| value.agent_id.as_deref()),+ "parent_agent_id": metadata.and_then(|value| value.parent_agent_id.as_deref()),+ "task_id": metadata.and_then(|value| value.task_id.as_deref()),+ "turn_id": metadata.and_then(|value| value.turn_id.as_deref()),+ "correlation_id": metadata.and_then(|value| value.correlation_id.as_deref()),+ })+}++#[cfg(test)]+mod tests {+ use std::collections::{BTreeMap, HashMap};++ use switchyard_llm_client::ClientRouter;+ use switchyard_protocol::{LlmClientError, LlmResponseStreamEvent, ModelId, Usage};+ use switchyard_runner::{AlgorithmSpec, ModelCapabilities, RunnerError};+ use wiremock::matchers::{body_partial_json, method, path};+ use wiremock::{Mock, MockServer, ResponseTemplate};++ use super::*;++ fn runtime_for(model: &str) -> SwitchyardRuntime {+ let algorithm = AlgorithmSpec::Noop {}+ .build("relay", &BTreeMap::new())+ .expect("noop route should build");+ let route = Route::new(+ algorithm,+ ClientRouter::new(HashMap::new()),+ None,+ ModelCapabilities::default(),+ None,+ None,+ Vec::new(),+ );+ SwitchyardRuntime {+ runner: Runner::new(vec![(ModelId::from(model), route)]),+ translation: TranslationEngine::default(),+ }+ }++ fn runtime_for_target(format: &str, base_url: &str) -> SwitchyardRuntime {+ let deployment = json!({+ "schema_version": 1,+ "llm_clients": {+ "target": {+ "format": format,+ "base_url": base_url,+ }+ },+ "targets": {+ "default": {+ "id": "target/model",+ "llm_client": "target",+ }+ },+ "routes": {+ "default": {+ "id": "switchyard/default",+ "type": "passthrough",+ "target": "default",+ }+ }+ });+ SwitchyardRuntime::new(crate::config::SwitchyardConfig {+ priority: 0,+ switchyard_config_path: None,+ switchyard_config: Some(deployment.as_object().unwrap().clone()),+ })+ .expect("cross-format runtime should load")+ }++ fn namespaced_responses_request() -> RelayRequest {+ RelayRequest {+ headers: Map::new(),+ content: json!({+ "model": "switchyard/default",+ "input": "Search the docs",+ "tools": [{+ "type": "namespace",+ "name": "mcp__docs",+ "tools": [{+ "type": "function",+ "name": "search",+ "parameters": {+ "type": "object",+ "properties": {"query": {"type": "string"}},+ "required": ["query"],+ },+ }],+ }],+ }),+ }+ }++ #[test]+ fn only_configured_route_models_are_managed() {+ let runtime = runtime_for("switchyard");+ assert!(runtime.manages_model("switchyard"));+ assert!(!runtime.manages_model("other"));+ }++ #[tokio::test]+ async fn responses_requests_use_the_configured_target_format() {+ for (format, endpoint, upstream_response) in [+ (+ "openai_chat",+ "/v1/chat/completions",+ json!({+ "id": "chatcmpl_test",+ "object": "chat.completion",+ "model": "target/model",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }],+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},+ }),+ ),+ (+ "anthropic_messages",+ "/v1/messages",+ json!({+ "id": "msg_test",+ "type": "message",+ "role": "assistant",+ "model": "target/model",+ "content": [{"type": "text", "text": "ok"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 1, "output_tokens": 1},+ }),+ ),+ ] {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path(endpoint))+ .and(body_partial_json(json!({"model": "target/model"})))+ .respond_with(ResponseTemplate::new(200).set_body_json(upstream_response))+ .expect(1)+ .mount(&server)+ .await;+ let runtime = runtime_for_target(format, &format!("{}/v1", server.uri()));+ let request = RelayRequest {+ headers: Map::new(),+ content: json!({+ "model": "switchyard/default",+ "input": "hello",+ }),+ };+ let decoded = runtime+ .decode_request(WireFormat::OpenAiResponses, request, false)+ .expect("Responses request should decode");+ assert_eq!(+ decoded+ .metadata+ .as_ref()+ .and_then(|metadata| metadata.wire_format),+ None+ );++ let execution = runtime+ .execute_buffered(WireFormat::OpenAiResponses, decoded)+ .await;+ let response = execution.result.expect("target call should succeed");+ assert_eq!(response["object"], "response");+ assert_eq!(response["model"], "target/model");+ }+ }++ #[tokio::test]+ async fn buffered_responses_restore_codex_tool_namespaces() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .and(body_partial_json(json!({"model": "target/model"})))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "id": "chatcmpl_tool",+ "object": "chat.completion",+ "model": "target/model",+ "choices": [{+ "index": 0,+ "message": {+ "role": "assistant",+ "content": null,+ "tool_calls": [{+ "id": "call_search",+ "type": "function",+ "function": {+ "name": "mcp__docs__search",+ "arguments": "{\"query\":\"routing\"}",+ },+ }],+ },+ "finish_reason": "tool_calls",+ }],+ "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},+ })))+ .expect(1)+ .mount(&server)+ .await;+ let runtime = runtime_for_target("openai_chat", &format!("{}/v1", server.uri()));+ let request = runtime+ .decode_request(+ WireFormat::OpenAiResponses,+ namespaced_responses_request(),+ false,+ )+ .expect("namespaced Responses request should decode");++ let response = runtime+ .execute_buffered(WireFormat::OpenAiResponses, request)+ .await+ .result+ .expect("buffered target call should succeed");+ let tool_call = response["output"]+ .as_array()+ .and_then(|output| output.iter().find(|item| item["type"] == "function_call"))+ .expect("Responses output should contain a function call");+ assert_eq!(tool_call["name"], "search");+ assert_eq!(tool_call["namespace"], "mcp__docs");+ }++ #[tokio::test]+ async fn streaming_responses_restore_codex_tool_namespaces() {+ let server = MockServer::start().await;+ let sse = concat!(+ "data: {\"id\":\"chatcmpl_tool\",\"object\":\"chat.completion.chunk\",\"model\":\"target/model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_search\",\"type\":\"function\",\"function\":{\"name\":\"mcp__docs__search\",\"arguments\":\"{\\\"query\\\":\\\"routing\\\"}\"}}]},\"finish_reason\":null}]}\n\n",+ "data: {\"id\":\"chatcmpl_tool\",\"object\":\"chat.completion.chunk\",\"model\":\"target/model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n",+ "data: [DONE]\n\n",+ );+ Mock::given(method("POST"))+ .and(path("/v1/chat/completions"))+ .and(body_partial_json(+ json!({"model": "target/model", "stream": true}),+ ))+ .respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))+ .expect(1)+ .mount(&server)+ .await;+ let runtime = runtime_for_target("openai_chat", &format!("{}/v1", server.uri()));+ let request = runtime+ .decode_request(+ WireFormat::OpenAiResponses,+ namespaced_responses_request(),+ true,+ )+ .expect("namespaced streaming Responses request should decode");++ let stream = runtime+ .execute_stream(WireFormat::OpenAiResponses, request, Arc::new(|_| {}))+ .await+ .result+ .expect("streaming target call should succeed");+ let events = stream+ .collect::<Vec<_>>()+ .await+ .into_iter()+ .collect::<Result<Vec<_>, _>>()+ .expect("response stream should encode");++ for event_type in ["response.output_item.added", "response.output_item.done"] {+ let item = events+ .iter()+ .find(|event| event["type"] == event_type)+ .map(|event| &event["item"])+ .unwrap_or_else(|| panic!("stream produced no {event_type}"));+ assert_eq!(item["name"], "search", "{event_type}");+ assert_eq!(item["namespace"], "mcp__docs", "{event_type}");+ }+ let completed = events+ .iter()+ .find(|event| event["type"] == "response.completed")+ .expect("stream produced no response.completed event");+ assert_eq!(completed["response"]["output"][0]["name"], "search");+ assert_eq!(completed["response"]["output"][0]["namespace"], "mcp__docs");+ }++ #[test]+ fn execution_failure_mark_uses_the_safe_runner_summary() {+ let secret = "provider response body";+ let error = RunnerError::Client(LlmClientError::ContextWindowExceeded {+ model: ModelId::from("weak"),+ message: secret.into(),+ });++ let mark = route_execution_error_mark(+ &error.execution_error_summary(),+ json!({"session_id": "session"}),+ );++ assert_eq!(mark.name, "switchyard.routing.error");+ assert_eq!(mark.data["failure_kind"], "route_execution");+ assert_eq!(mark.data["category"], "context_window_exceeded");+ assert_eq!(mark.data["phase"], "before_response");+ assert_eq!(mark.data["target"], "weak");+ assert_eq!(mark.data["upstream_status"], Json::Null);+ assert_eq!(mark.severity, Some(LogSeverity::Error));+ assert!(!mark.data.to_string().contains(secret));+ }++ #[test]+ fn routing_observations_emit_debug_marks_and_metrics() {+ let runtime = runtime_for("switchyard");+ let mut events = Vec::new();+ runtime.emit_observations(+ &mut events,+ vec![+ RunObservation::LlmCall(LlmCallObservation {+ selected_model: ModelId::from("routing-model"),+ is_success: false,+ duration: std::time::Duration::from_millis(12),+ usage: Some(Usage {+ input_tokens: Some(4),+ ..Usage::default()+ }),+ }),+ RunObservation::RoutingOverhead(std::time::Duration::from_millis(3)),+ ],+ &json!({"session_id": "session"}),+ );++ assert_eq!(events.len(), 6);+ let RoutingEvent::Mark(call_mark) = &events[0] else {+ panic!("first event should be the routing call mark");+ };+ assert_eq!(call_mark.name, "switchyard.routing.llm_call");+ assert_eq!(call_mark.severity, Some(LogSeverity::Debug));+ assert_eq!(call_mark.data["outcome"], "error");+ assert!(call_mark.data.get("usage").is_none());++ let RoutingEvent::Metric(call_count) = &events[1] else {+ panic!("second event should be the routing call counter");+ };+ assert_eq!(call_count.name, "switchyard.routing.llm_calls");+ assert_eq!(call_count.measurements[0].kind, MetricKind::Counter);+ assert_eq!(+ call_count.measurements[0].attributes,+ Some(json!({"outcome": "error"}))+ );++ let RoutingEvent::Metric(call_duration) = &events[2] else {+ panic!("third event should be the routing call histogram");+ };+ assert_eq!(call_duration.name, "switchyard.routing.llm_call.duration");+ assert_eq!(call_duration.measurements[0].kind, MetricKind::Histogram);+ assert_eq!(call_duration.measurements[0].value, json!(12.0));++ let RoutingEvent::Metric(tokens) = &events[3] else {+ panic!("fourth event should be the routing token counter");+ };+ assert_eq!(tokens.name, "switchyard.routing.llm_tokens");++ let RoutingEvent::Mark(overhead_mark) = &events[4] else {+ panic!("fifth event should be the routing overhead mark");+ };+ assert_eq!(overhead_mark.severity, Some(LogSeverity::Info));++ let RoutingEvent::Metric(overhead) = &events[5] else {+ panic!("sixth event should be the routing overhead histogram");+ };+ assert_eq!(overhead.name, "switchyard.routing.overhead");+ assert_eq!(overhead.measurements[0].attributes, Some(json!({})));+ }++ #[test]+ fn request_and_failure_metrics_use_bounded_attributes() {+ let RoutingEvent::Metric(request) = request_metric("stage_router", json!({})) else {+ panic!("request should be a metric");+ };+ assert_eq!(request.name, "switchyard.routing.requests");+ assert_eq!(+ request.measurements[0].attributes,+ Some(json!({"algorithm": "stage_router"}))+ );++ let RoutingEvent::Metric(failure) = failure_metric(+ "route_execution",+ Some("upstream_http"),+ Some("before_response"),+ Some(503),+ json!({}),+ ) else {+ panic!("failure should be a metric");+ };+ assert_eq!(failure.name, "switchyard.routing.failures");+ assert_eq!(+ failure.measurements[0].attributes,+ Some(json!({+ "failure_kind": "route_execution",+ "category": "upstream_http",+ "phase": "before_response",+ "upstream_status": 503,+ }))+ );+ }++ #[test]+ fn token_usage_metrics_distinguish_routing_and_answer_targets() {+ let call = LlmCallObservation {+ selected_model: ModelId::from("judge-model"),+ is_success: true,+ duration: std::time::Duration::from_millis(1),+ usage: Some(Usage {+ input_tokens: Some(11),+ cache: Usage::cache_details(Some(3), Some(2)),+ output_tokens: Some(7),+ total_tokens: Some(23),+ reasoning_tokens: Some(5),+ }),+ };++ let routing = token_usage_metrics("routing", &call, &json!({"session_id": "session"}));+ assert_eq!(routing.len(), 6);+ for event in &routing {+ let RoutingEvent::Metric(metric) = event else {+ panic!("token usage should be emitted as a metric");+ };+ assert_eq!(metric.name, "switchyard.routing.llm_tokens");+ assert_eq!(metric.measurements[0].kind, MetricKind::Counter);+ assert_eq!(metric.measurements[0].unit.as_deref(), Some("{token}"));+ assert_eq!(+ metric.measurements[0].attributes.as_ref().unwrap()["call_role"],+ "routing"+ );+ assert_eq!(+ metric.measurements[0].attributes.as_ref().unwrap()["target_model"],+ "judge-model"+ );+ }+ let token_values = routing+ .iter()+ .map(|event| {+ let RoutingEvent::Metric(metric) = event else {+ panic!("token usage should be emitted as a metric");+ };+ metric.measurements[0].value.clone()+ })+ .collect::<Vec<_>>();+ assert_eq!(+ token_values,+ vec![json!(11), json!(3), json!(2), json!(7), json!(5), json!(23)]+ );++ let answer = token_usage_metrics("answer", &call, &json!({}));+ let RoutingEvent::Metric(metric) = &answer[0] else {+ panic!("answer usage should be emitted as a metric");+ };+ assert_eq!(+ metric.measurements[0].attributes.as_ref().unwrap()["call_role"],+ "answer"+ );+ }++ #[test]+ fn answer_observations_emit_token_metrics_without_answer_logs() {+ let runtime = runtime_for("switchyard");+ let mut events = Vec::new();+ runtime.emit_observations(+ &mut events,+ vec![RunObservation::AnswerCall(LlmCallObservation {+ selected_model: ModelId::from("selected-target"),+ is_success: true,+ duration: std::time::Duration::from_millis(2),+ usage: Some(Usage {+ output_tokens: Some(9),+ ..Usage::default()+ }),+ })],+ &json!({}),+ );++ assert_eq!(events.len(), 1);+ let RoutingEvent::Metric(metric) = &events[0] else {+ panic!("answer observation should only emit a token metric");+ };+ assert_eq!(metric.name, "switchyard.routing.llm_tokens");+ assert_eq!(+ metric.measurements[0].attributes,+ Some(json!({+ "call_role": "answer",+ "target_model": "selected-target",+ "token_type": "output",+ }))+ );+ }++ #[tokio::test]+ async fn stream_usage_emits_answer_token_metrics() {+ let response = Response {+ llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([+ Ok(LlmResponseStreamEvent::new(vec![+ LlmResponseChunk::MessageStop { reason: None },+ ])),+ Ok(LlmResponseStreamEvent::new(vec![LlmResponseChunk::Usage(+ Usage {+ input_tokens: Some(10),+ output_tokens: Some(3),+ total_tokens: Some(13),+ ..Usage::default()+ },+ )])),+ ]))),+ metadata: Some(Metadata {+ served_model: Some(ModelId::from("selected-target")),+ ..Default::default()+ }),+ };+ let captured = Arc::new(Mutex::new(Vec::new()));+ let emitted = Arc::clone(&captured);+ let mut stream = returned_events(+ response,+ WireFormat::OpenAiChat,+ &ProviderExtensions::default(),+ json!({"session_id": "session"}),+ Arc::new(move |event| emitted.lock().unwrap().push(event)),+ )+ .expect("stream setup should succeed");++ assert!(captured.lock().unwrap().is_empty());+ assert!(stream.next().await.expect("encoded stream event").is_ok());+ assert!(captured.lock().unwrap().is_empty());+ assert!(stream.next().await.expect("encoded usage event").is_ok());++ let events = captured.lock().unwrap();+ assert_eq!(events.len(), 3);+ let token_types = events+ .iter()+ .map(|event| {+ let RoutingEvent::Metric(metric) = event else {+ panic!("stream usage should emit only metrics");+ };+ assert_eq!(metric.name, "switchyard.routing.llm_tokens");+ assert_eq!(metric.metadata, json!({"session_id": "session"}));+ let attributes = metric.measurements[0].attributes.as_ref().unwrap();+ assert_eq!(attributes["call_role"], "answer");+ assert_eq!(attributes["target_model"], "selected-target");+ attributes["token_type"].as_str().unwrap().to_string()+ })+ .collect::<Vec<_>>();+ assert_eq!(token_types, ["input", "output", "total"]);+ }++ #[tokio::test]+ async fn aggregate_response_stream_does_not_reemit_usage_metrics() {+ let mut aggregate =+ switchyard_protocol::text_response(Some("selected-target".into()), "ok");+ aggregate.usage = Usage {+ input_tokens: Some(10),+ output_tokens: Some(3),+ total_tokens: Some(13),+ ..Usage::default()+ };+ let response = Response {+ llm_response: LlmResponse::Agg(aggregate),+ metadata: Some(Metadata {+ served_model: Some(ModelId::from("selected-target")),+ ..Default::default()+ }),+ };+ let captured = Arc::new(Mutex::new(Vec::new()));+ let emitted = Arc::clone(&captured);+ let stream = returned_events(+ response,+ WireFormat::OpenAiChat,+ &ProviderExtensions::default(),+ json!({}),+ Arc::new(move |event| emitted.lock().unwrap().push(event)),+ )+ .expect("stream setup should succeed");++ let events = stream.collect::<Vec<_>>().await;+ assert!(events.iter().all(Result::is_ok));+ assert!(captured.lock().unwrap().is_empty());+ }++ #[tokio::test]+ async fn stream_failure_emits_a_safe_failure_mark() {+ let secret = "provider response body";+ let response = Response {+ llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([Err::<+ LlmResponseStreamEvent,+ LlmClientError,+ >(+ LlmClientError::ContextWindowExceeded {+ model: ModelId::from("weak"),+ message: secret.into(),+ },+ )]))),+ metadata: Some(Metadata {+ served_model: Some(ModelId::from("strong")),+ ..Default::default()+ }),+ };+ let captured = Arc::new(Mutex::new(Vec::new()));+ let emitted = Arc::clone(&captured);+ let stream = returned_events(+ response,+ WireFormat::OpenAiChat,+ &ProviderExtensions::default(),+ json!({"session_id": "session"}),+ Arc::new(move |mark| emitted.lock().unwrap().push(mark)),+ )+ .expect("stream setup should succeed");++ let events = stream.collect::<Vec<_>>().await;+ assert!(events[0].is_err());+ assert!(+ !events[0]+ .as_ref()+ .expect_err("stream should fail")+ .contains(secret)+ );+ let events = captured.lock().unwrap();+ assert_eq!(events.len(), 2);+ let RoutingEvent::Mark(mark) = &events[0] else {+ panic!("first event should be the safe failure mark");+ };+ assert_eq!(mark.data["category"], "context_window_exceeded");+ assert_eq!(mark.data["phase"], "during_stream");+ assert_eq!(mark.data["target"], "strong");+ assert_eq!(mark.severity, Some(LogSeverity::Error));+ assert!(!mark.data.to_string().contains(secret));+ let RoutingEvent::Metric(metric) = &events[1] else {+ panic!("second event should be the failure counter");+ };+ assert_eq!(metric.name, "switchyard.routing.failures");+ assert_eq!(+ metric.measurements[0].attributes,+ Some(json!({+ "failure_kind": "route_execution",+ "category": "context_window_exceeded",+ "phase": "during_stream",+ }))+ );+ }++ async fn assert_normalized_stream_failure(+ chunk: LlmResponseChunk,+ expected_category: &str,+ expected_status: Option<u16>,+ ) {+ let secret = "provider response body";+ let response = Response {+ llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([Ok(+ LlmResponseStreamEvent::new(vec![chunk]),+ )]))),+ metadata: Some(Metadata {+ served_model: Some(ModelId::from("selected-target")),+ ..Default::default()+ }),+ };+ let captured = Arc::new(Mutex::new(Vec::new()));+ let emitted = Arc::clone(&captured);+ let stream = returned_events(+ response,+ WireFormat::OpenAiChat,+ &ProviderExtensions::default(),+ json!({"session_id": "session"}),+ Arc::new(move |event| emitted.lock().unwrap().push(event)),+ )+ .expect("stream setup should succeed");++ let client_events = stream.collect::<Vec<_>>().await;+ assert_eq!(client_events.len(), 1);+ assert!(client_events[0].is_err());+ assert!(+ !client_events[0]+ .as_ref()+ .expect_err("stream should fail")+ .contains(secret)+ );++ let events = captured.lock().unwrap();+ assert_eq!(events.len(), 2, "one mark and one counter should emit");+ let RoutingEvent::Mark(mark) = &events[0] else {+ panic!("first event should be the safe failure mark");+ };+ assert_eq!(mark.data["category"], expected_category);+ assert_eq!(mark.data["phase"], "during_stream");+ assert_eq!(+ mark.data["upstream_status"],+ expected_status.map_or(Json::Null, Json::from)+ );+ assert_eq!(mark.data["target"], "selected-target");+ assert!(!mark.data.to_string().contains(secret));+ let RoutingEvent::Metric(metric) = &events[1] else {+ panic!("second event should be the failure counter");+ };+ assert_eq!(metric.name, "switchyard.routing.failures");+ let mut expected_attributes = json!({+ "failure_kind": "route_execution",+ "category": expected_category,+ "phase": "during_stream",+ });+ if let Some(status) = expected_status {+ expected_attributes+ .as_object_mut()+ .expect("expected metric attributes should be an object")+ .insert("upstream_status".into(), json!(status));+ }+ assert_eq!(metric.measurements[0].attributes, Some(expected_attributes));+ }++ #[tokio::test]+ async fn normalized_stream_error_emits_failure_telemetry_once() {+ assert_normalized_stream_failure(+ LlmResponseChunk::StreamError {+ message: "provider response body".into(),+ },+ "upstream_http",+ Some(502),+ )+ .await;+ }++ #[tokio::test]+ async fn normalized_decode_error_emits_failure_telemetry_once() {+ assert_normalized_stream_failure(+ LlmResponseChunk::DecodeError {+ message: "provider response body".into(),+ },+ "response_translation",+ None,+ )+ .await;+ }+}diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rsnew file mode 100644index 000000000..b565d5b58--- /dev/null+++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs@@ -0,0 +1,66 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use nemo_relay_plugin::LlmRequest as RelayRequest;+use serde_json::Value as Json;+use switchyard_protocol::{AggLlmResponse, LlmRequest, ProviderExtensions, WireFormat};+use switchyard_translation::{+ DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy,+ TargetCapabilities, TranslationDiagnostic, TranslationEngine, TranslationPolicy,+ UnknownFieldPolicy,+};++pub(crate) fn decode_request(+ engine: &TranslationEngine,+ protocol: WireFormat,+ request: &RelayRequest,+) -> Result<LlmRequest, String> {+ let output = engine+ .decode_request(protocol, &request.content, &policy())+ .map_err(error)?;+ safe(&output.diagnostics)?;+ Ok(output.request)+}++pub(crate) fn encode_response(+ engine: &TranslationEngine,+ protocol: WireFormat,+ response: &AggLlmResponse,+ request_extensions: &ProviderExtensions,+) -> Result<Json, String> {+ let output = engine+ .encode_response_with_extensions(protocol, response, request_extensions, &policy())+ .map_err(error)?;+ safe(&output.diagnostics)?;+ Ok(output.body)+}++fn policy() -> TranslationPolicy {+ TranslationPolicy {+ unknown_field_policy: UnknownFieldPolicy::Preserve,+ lossy_conversion_policy: LossyConversionPolicy::Reject,+ deterministic_ids: DeterministicIdPolicy::GenerateStable {+ prefix: "relay".into(),+ },+ preservation: PreservationPolicy::InMemory,+ target_capabilities: TargetCapabilities::default(),+ }+}++fn safe(diagnostics: &[TranslationDiagnostic]) -> Result<(), String> {+ let unsafe_diagnostics = diagnostics+ .iter()+ .filter(|diagnostic| diagnostic.severity != DiagnosticSeverity::Info)+ .collect::<Vec<_>>();+ if unsafe_diagnostics.is_empty() {+ Ok(())+ } else {+ Err(format!(+ "Switchyard translation was not lossless: {unsafe_diagnostics:?}"+ ))+ }+}++fn error(error: switchyard_translation::TranslationError) -> String {+ format!("Switchyard translation failed: {error}")+}diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rsindex 9761a46af..abd30955c 100644--- a/crates/switchyard-translation/src/engine.rs+++ b/crates/switchyard-translation/src/engine.rs@@ -19,7 +19,7 @@ use crate::codecs::stream::{use crate::diagnostic::TranslationDiagnostic;use crate::error::{Result, TranslationError};use crate::format::FormatId;-use crate::llm::{AggLlmResponse, LlmRequest};+use crate::llm::{AggLlmResponse, LlmRequest, ProviderExtensions};use crate::policy::TranslationPolicy;/// Encoded translation result with any diagnostics emitted along the way.@@ -201,6 +201,25 @@ impl TranslationEngine {})}+ /// Encodes a neutral response IR while applying metadata retained from its request.+ ///+ /// Request extensions are used to restore caller-specific response details, such as+ /// Codex tool namespaces, without exposing those details to the upstream provider.+ pub fn encode_response_with_extensions(+ &self,+ target: impl Into<FormatId>,+ response: &AggLlmResponse,+ request_extensions: &ProviderExtensions,+ policy: &TranslationPolicy,+ ) -> Result<TranslationOutput> {+ let mut output = self.encode_response(target, response, policy)?;+ crate::codex_namespaces::restore_qualified_tool_names(+ &mut output.body,+ &crate::codex_namespaces::qualified_tool_origins(request_extensions),+ );+ Ok(output)+ }+/// Translates a response body from source format to target format.pub fn translate_response(&self,diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rsindex a1cdd1d3c..061517e24 100644--- a/crates/switchyard-translation/src/helpers.rs+++ b/crates/switchyard-translation/src/helpers.rs@@ -76,15 +76,16 @@ pub fn encode_aggregated_response_with_extensions(request_extensions: &switchyard_protocol::ProviderExtensions,) -> Result<Value> {let mut body = DEFAULT_TRANSLATION_ENGINE- .encode_response(wire_format, agg, &DEFAULT_TRANSLATION_POLICY)?+ .encode_response_with_extensions(+ wire_format,+ agg,+ request_extensions,+ &DEFAULT_TRANSLATION_POLICY,+ )?.body;if let (Some(model), Value::Object(object)) = (served_model, &mut body) {object.insert("model".to_string(), Value::String(model.to_string()));}- crate::codex_namespaces::restore_qualified_tool_names(- &mut body,- &crate::codex_namespaces::qualified_tool_origins(request_extensions),- );Ok(body)}diff --git a/docs/index.md b/docs/index.mdindex e68ed477b..9b8fb17f9 100644--- a/docs/index.md+++ b/docs/index.md@@ -9,6 +9,7 @@ It supports OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages.|---|---|---|| Run Switchyard as a standalone proxy for API clients | Server Path | [Build and run the Rust server](getting_started.md#server-path) || Add Switchyard routing to a Rust application | Library Path | [`switchyard-libsy`](../crates/libsy/README.md) |+| Add Switchyard routing to NeMo Relay | Native Plugin Path | [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md) |The Server Path builds and runs the standalone `switchyard-server` binary.@@ -27,3 +28,4 @@ The Server Path builds and runs the standalone `switchyard-server` binary.- [`switchyard-libsy`](reference/rust_api.md#switchyard-libsy): embeddable routing algorithms- [`switchyard-protocol`](reference/rust_api.md#switchyard-protocol): provider-neutral API types- [`switchyard-translation`](../crates/switchyard-translation/README.md): protocol translation+- [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md): native NeMo Relay integrationdiff --git a/tests/relay_plugin/test_package_bundle.py b/tests/relay_plugin/test_package_bundle.pynew file mode 100644index 000000000..05da044d8--- /dev/null+++ b/tests/relay_plugin/test_package_bundle.py@@ -0,0 +1,106 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Tests for the native Relay plugin bundle packager."""++from __future__ import annotations++import hashlib+import subprocess+import sys+import tarfile+import tempfile+import unittest+import zipfile+from pathlib import Path++REPOSITORY_ROOT = Path(__file__).resolve().parents[2]+CRATE_ROOT = REPOSITORY_ROOT / "crates" / "switchyard-nemo-relay-plugin"+PACKAGER = CRATE_ROOT / "scripts" / "package_bundle.py"+PACKAGE_NAME = "switchyard-nemo-relay-plugin"+++class PackageBundleTest(unittest.TestCase):+ """Verify materialized and archived plugin bundle contents."""++ def test_materializes_and_archives_supported_formats(self) -> None:+ for archive_suffix in (".tar.gz", ".zip"):+ with self.subTest(archive_suffix=archive_suffix), tempfile.TemporaryDirectory() as tmp:+ root = Path(tmp)+ library = root / "libswitchyard_nemo_relay_plugin.so"+ library.write_bytes(b"compiled plugin")+ output = root / "bundle"+ archive = root / f"{PACKAGE_NAME}-0.2.0-linux-x86_64{archive_suffix}"++ subprocess.run(+ [+ sys.executable,+ str(PACKAGER),+ "--library",+ str(library),+ "--output",+ str(output),+ "--archive",+ str(archive),+ ],+ check=True,+ capture_output=True,+ text=True,+ )++ expected = {+ "LICENSE",+ "NOTICE",+ "config.schema.json",+ library.name,+ "relay-plugin.toml",+ }+ self.assertEqual({path.name for path in output.iterdir()}, expected)+ manifest = (output / "relay-plugin.toml").read_text(encoding="utf-8")+ self.assertIn(f'artifact = "{library.name}"', manifest)+ self.assertIn(hashlib.sha256(library.read_bytes()).hexdigest(), manifest)+ self.assertNotIn("<platform-library-file>", manifest)+ self.assertNotIn("<artifact-sha256>", manifest)+ self.assertEqual(self.archive_members(archive), {f"{PACKAGE_NAME}/{name}" for name in expected})++ def test_rejects_archive_inside_output_before_materializing_bundle(self) -> None:+ with tempfile.TemporaryDirectory() as tmp:+ root = Path(tmp)+ library = root / "libswitchyard_nemo_relay_plugin.so"+ library.write_bytes(b"compiled plugin")+ output = root / "bundle"+ archive = output / "plugin.zip"++ result = subprocess.run(+ [+ sys.executable,+ str(PACKAGER),+ "--library",+ str(library),+ "--output",+ str(output),+ "--archive",+ str(archive),+ ],+ check=False,+ capture_output=True,+ text=True,+ )++ self.assertNotEqual(result.returncode, 0)+ self.assertIn("bundle archive must be outside output directory", result.stderr)+ self.assertFalse(output.exists())+ self.assertFalse(archive.exists())++ @staticmethod+ def archive_members(archive: Path) -> set[str]:+ """Return regular-file paths from a supported bundle archive."""+ if archive.name.endswith(".tar.gz"):+ with tarfile.open(archive) as stream:+ return {member.name for member in stream.getmembers() if member.isfile()}+ with zipfile.ZipFile(archive) as stream:+ return {member.filename for member in stream.infolist() if not member.is_dir()}+++if __name__ == "__main__":+ unittest.main()