Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): redact provider API keys from client-facing responses (#704)
Graham Kingbecbda5d51ad
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex e2e1c7b14..60a8f3444 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -207,7 +207,8 @@ impl DeploymentConfig {}}- let clients = self.build_clients()?;+ let mut provider_api_keys = Vec::new();+ let clients = self.build_clients(&mut provider_api_keys)?;let targets = self.build_targets();let fallback_base_url = self.fallback_base_url()?;let mut routes = Vec::with_capacity(self.routes.len());@@ -260,11 +261,16 @@ impl DeploymentConfig {);routes.push((config.id.clone(), route));}- let runner = Runner::new(routes).with_fallback_url(fallback_base_url);+ let runner = Runner::new(routes)+ .with_fallback_url(fallback_base_url)+ .with_provider_api_keys(provider_api_keys);Ok(runner)}- fn build_clients(&self) -> RunnerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {+ fn build_clients(+ &self,+ provider_api_keys: &mut Vec<String>,+ ) -> RunnerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {let mut models_by_client = self.llm_clients.keys()@@ -273,7 +279,13 @@ impl DeploymentConfig {for (name, client_config) in &self.llm_clients {validate_value("llm client name", name)?;- build_backend(name, client_config, &BTreeMap::new(), None)?;+ let backend = build_backend(name, client_config, &BTreeMap::new(), None)?;+ let (Backend::OpenAiChat(config)+ | Backend::OpenAiResponses(config)+ | Backend::Anthropic(config)) = backend;+ if let Some(key) = config.api_key {+ provider_api_keys.push(key);+ }}for (target_name, target) in &self.targets {let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rsindex 36ecb6410..420df2ccc 100644--- a/crates/switchyard-runner/src/runner.rs+++ b/crates/switchyard-runner/src/runner.rs@@ -17,6 +17,7 @@ use crate::{ModelCapabilities, Route, RunnerError};pub struct Runner {routes: Vec<(ModelId, Route)>,fallback_base_url: Option<String>,+ provider_api_keys: Vec<String>,}/// Borrowed model metadata returned while listing routes.@@ -61,9 +62,22 @@ impl Runner {Self {routes,fallback_base_url: None,+ provider_api_keys: Vec::new(),}}+ /// Registers deployment-owned API keys for server response redaction.+ /// TOML loading registers these automatically; programmatic hosts must supply them.+ pub fn with_provider_api_keys(mut self, keys: Vec<String>) -> Self {+ self.provider_api_keys = keys;+ self+ }++ /// Returns deployment-owned secrets for the server's response redactor.+ pub fn provider_api_keys(&self) -> &[String] {+ &self.provider_api_keys+ }+pub(crate) fn with_fallback_url(mut self, fallback_base_url: Option<String>) -> Self {self.fallback_base_url = fallback_base_url;selfdiff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex dce710d35..6525e8c52 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -6,6 +6,7 @@pub mod config;mod metrics;mod observability;+mod redaction;mod response;mod routing_log;mod shutdown;@@ -157,6 +158,7 @@ struct DecisionLlmClientResponse {#[derive(Clone)]pub struct ServerState {runner: Arc<Runner>,+ redactor: Arc<redaction::Redactor>,fallback_http: reqwest::Client,metrics: prometheus::Registry,stats: StatsAccumulator,@@ -210,7 +212,9 @@ impl ServerState {metrics.clone(),runner.models().map(|model| model.algorithm),);+ let redactor = redaction::Redactor::new(runner.provider_api_keys());Ok(Self {+ redactor: Arc::new(redactor),runner: Arc::new(runner),fallback_http,metrics,@@ -518,6 +522,10 @@ fn primary_llm_routes() -> Router<ServerState> {fn finish_router(router: Router<ServerState>, state: ServerState) -> Router {router.layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES))+ .layer(axum::middleware::from_fn_with_state(+ state.clone(),+ redaction::redact_response,+ ))// `layer` only wraps routes registered before it, so this stays last..layer(axum::middleware::from_fn(stamp_request_start)).with_state(state)@@ -1060,11 +1068,16 @@ async fn handle_llm_request(let upstream_headers = std::mem::take(&mut response.upstream_headers);let response_model = served_model.as_ref().map(ToString::to_string);- let mut response =- match into_http_response(response, wire_format, response_model, request_extensions) {- Ok(response) => response,- Err(error) => return server_error(error.to_string()),- };+ let mut response = match into_http_response(+ response,+ wire_format,+ response_model,+ request_extensions,+ Arc::clone(&state.redactor),+ ) {+ Ok(response) => response,+ Err(error) => return server_error(error.to_string()),+ };// Forward upstream headers before Switchyard writes its own so any header// this server emits always overrides an upstream echo of the same name.let response_headers = response.headers_mut();diff --git a/crates/switchyard-server/src/redaction.rs b/crates/switchyard-server/src/redaction.rsnew file mode 100644index 000000000..cff0df75b--- /dev/null+++ b/crates/switchyard-server/src/redaction.rs@@ -0,0 +1,221 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Removes configured provider credentials from client-facing responses.++use axum::body::{Body, to_bytes};+use axum::extract::{Request, State};+use axum::http::header::{CONTENT_ENCODING, CONTENT_LENGTH, CONTENT_TYPE};+use axum::middleware::Next;+use axum::response::Response;++use crate::{DEFAULT_MAX_REQUEST_BODY_BYTES, ServerState};++#[derive(Default)]+pub(crate) struct Redactor {+ raw: Vec<String>,+ json: Vec<String>,+}++impl Redactor {+ pub(crate) fn new(keys: &[String]) -> Self {+ let mut raw: Vec<String> = keys.iter().filter(|key| !key.is_empty()).cloned().collect();+ raw.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b)));+ raw.dedup();+ let mut json = Vec::with_capacity(raw.len());+ for key in &raw {+ let Ok(encoded) = serde_json::to_string(key) else {+ unreachable!("serde_json::to_string on a String cannot fail");+ };+ json.push(encoded[1..encoded.len() - 1].to_string());+ }+ json.sort_by_key(|key| std::cmp::Reverse(key.len()));+ Self { raw, json }+ }++ pub(crate) fn json(&self, value: String) -> String {+ replace(value, &self.json)+ }++ pub(crate) fn text(&self, value: String) -> String {+ replace(value, &self.raw)+ }+}++fn replace(mut value: String, secrets: &[String]) -> String {+ for secret in secrets {+ if value.contains(secret) {+ value = value.replace(secret, "[REDACTED]");+ }+ }+ value+}++pub(crate) async fn redact_response(+ State(state): State<ServerState>,+ request: Request,+ next: Next,+) -> Response {+ let mut response = next.run(request).await;+ let redactor = &state.redactor;+ if redactor.raw.is_empty() {+ return response;+ }+ let is_json = response.headers().get(CONTENT_TYPE).is_some_and(|value| {+ value.to_str().is_ok_and(|value| {+ value.split(';').next().is_some_and(|mime| {+ mime.trim() == "application/json" || mime.trim().ends_with("+json")+ })+ })+ });+ let is_encoded = response+ .headers()+ .get_all(CONTENT_ENCODING)+ .iter()+ .any(|value| {+ value.to_str().map_or(true, |value| {+ value+ .split(',')+ .any(|encoding| !encoding.trim().eq_ignore_ascii_case("identity"))+ })+ });+ for value in response.headers_mut().values_mut() {+ if let Ok(text) = value.to_str() {+ let sanitized = redactor.text(text.to_string());+ if sanitized != text {+ // Replacement contains only visible ASCII and cannot invalidate a header.+ if let Ok(header) = sanitized.parse() {+ *value = header;+ }+ }+ }+ }+ // SSE bodies are redacted per event before framing, without buffering the stream.+ // The fallback proxy passes compressed bodies through without decoding them.+ if is_json && !is_encoded {+ let (mut parts, body) = response.into_parts();+ let body = match to_bytes(body, DEFAULT_MAX_REQUEST_BODY_BYTES).await {+ Ok(bytes) => match String::from_utf8(bytes.to_vec()) {+ Ok(json) => Body::from(redactor.json(json)),+ Err(_) => return crate::server_error("Invalid upstream JSON"),+ },+ Err(_) => return crate::server_error("Unable to read response body"),+ };+ parts.headers.remove(CONTENT_LENGTH);+ response = Response::from_parts(parts, body);+ }+ response+}++#[cfg(test)]+mod tests {+ use std::sync::Arc;++ use axum::Json;+ use axum::response::IntoResponse;+ use axum::routing::get;+ use serde_json::json;+ use switchyard_runner::Runner;+ use switchyard_translation::{LlmStreamError, WireFormat};+ use tower::ServiceExt;++ use super::*;++ #[tokio::test]+ async fn responses_redact_provider_keys_without_changing_errors()+ -> Result<(), Box<dyn std::error::Error + Send + Sync>> {+ let key = "synthetic-\"provider\\key";+ let error = json!({"error": {+ "message": "upstream failed",+ "type": "provider_error",+ "debug": format!("Bearer {key}"),+ key: [key, "ordinary diagnostics"]+ }});+ let runner = Runner::new(Vec::new()).with_provider_api_keys(vec![key.to_string()]);+ let state = ServerState::from_runner(runner)?;+ let buffered = error.clone();+ let streamed = error.clone();+ let header = axum::http::HeaderValue::from_str(key)?;+ // gzip-compressed {"ok":true}.+ const GZIP_JSON: &[u8] = &[+ 31, 139, 8, 0, 0, 0, 0, 0, 2, 3, 171, 86, 202, 207, 86, 178, 42, 41, 42, 77, 173, 5, 0,+ 144, 95, 212, 167, 11, 0, 0, 0,+ ];+ let router = axum::Router::new()+ .route(+ "/compressed",+ get(|| async {+ (+ [+ (CONTENT_TYPE, "application/json"),+ (CONTENT_ENCODING, "gzip"),+ (CONTENT_LENGTH, "31"),+ ],+ GZIP_JSON,+ )+ }),+ )+ .route("/buffered", get(move || async move { Json(buffered) }))+ .route(+ "/stream",+ get(move |State(state): State<ServerState>| async move {+ let events = futures_util::stream::iter([+ Ok(json!({"choices": [], "model": "ordinary-model"})),+ Err(LlmStreamError::Upstream(streamed)),+ ]);+ let mut response = crate::sse::frame_stream(+ Box::pin(events),+ WireFormat::OpenAiChat,+ Arc::clone(&state.redactor),+ )+ .into_response();+ response.headers_mut().insert("x-upstream-debug", header);+ response+ }),+ );+ let app = crate::finish_router(router, state);+ let compressed = app+ .clone()+ .oneshot(Request::builder().uri("/compressed").body(Body::empty())?)+ .await?;+ assert_eq!(compressed.status(), axum::http::StatusCode::OK);+ assert_eq!(compressed.headers()[CONTENT_ENCODING], "gzip");+ assert_eq!(compressed.headers()[CONTENT_LENGTH], "31");+ assert_eq!(+ to_bytes(compressed.into_body(), usize::MAX).await?.as_ref(),+ GZIP_JSON+ );+ for path in ["/buffered", "/stream"] {+ let response = app+ .clone()+ .oneshot(Request::builder().uri(path).body(Body::empty())?)+ .await?;+ if path == "/stream" {+ assert_eq!(response.headers()["x-upstream-debug"], "[REDACTED]");+ }+ let body =+ String::from_utf8(to_bytes(response.into_body(), usize::MAX).await?.to_vec())?;+ let error: serde_json::Value = if path == "/stream" {+ assert!(body.contains("ordinary-model"));+ assert!(!body.contains("[DONE]"));+ let data = body+ .lines()+ .filter_map(|line| line.strip_prefix("data: "))+ .next_back()+ .ok_or("missing SSE error")?;+ serde_json::from_str(data)?+ } else {+ serde_json::from_str(&body)?+ };+ assert_eq!(error["error"]["message"], "upstream failed");+ assert_eq!(error["error"]["type"], "provider_error");+ assert_eq!(error["error"]["debug"], "Bearer [REDACTED]");+ assert_eq!(+ error["error"]["[REDACTED]"],+ json!(["[REDACTED]", "ordinary diagnostics"])+ );+ assert!(!body.contains("synthetic-"));+ }+ Ok(())+ }+}diff --git a/crates/switchyard-server/src/response.rs b/crates/switchyard-server/src/response.rsindex 950ec6399..84dac5b41 100644--- a/crates/switchyard-server/src/response.rs+++ b/crates/switchyard-server/src/response.rs@@ -4,6 +4,7 @@//! Response encoding glue for libsy server endpoints.use std::error::Error;+use std::sync::Arc;use axum::Json;use axum::response::{IntoResponse, Response as HttpResponse};@@ -24,6 +25,7 @@ pub(crate) fn into_http_response(target_format: WireFormat,served_model: Option<String>,request_extensions: ProviderExtensions,+ redactor: Arc<crate::redaction::Redactor>,) -> Result<HttpResponse, BoxError> {match response.llm_response {LlmResponse::Agg(response) => {@@ -42,7 +44,7 @@ pub(crate) fn into_http_response(served_model,&request_extensions,)?;- Ok(frame_stream(events, target_format).into_response())+ Ok(frame_stream(events, target_format, redactor).into_response())}}}diff --git a/crates/switchyard-server/src/sse.rs b/crates/switchyard-server/src/sse.rsindex d94cb220e..e685ba5e5 100644--- a/crates/switchyard-server/src/sse.rs+++ b/crates/switchyard-server/src/sse.rs@@ -4,12 +4,15 @@//! SSE framing helpers for OpenAI, Anthropic, and Responses endpoints.use std::convert::Infallible;+use std::sync::Arc;use axum::response::sse::{Event, Sse};use futures_util::Stream;use serde_json::{Value, json};use switchyard_translation::{LlmStreamError, RawEventStream, WireFormat};+use crate::redaction::Redactor;+/// Boxed stream type accepted by Axum's SSE response wrapper.pub(crate) type SseFrameStream =std::pin::Pin<Box<dyn Stream<Item = Result<Event, Infallible>> + Send>>;@@ -18,32 +21,33 @@ pub(crate) type SseFrameStream =pub(crate) fn frame_stream(stream: RawEventStream,target_format: WireFormat,+ redactor: Arc<Redactor>,) -> Sse<SseFrameStream> {let framed = async_stream::stream! {let mut stream = stream;let mut failed = false;while let Some(item) = futures_util::StreamExt::next(&mut stream).await {let event = match item {- Ok(value) => match frame_event(target_format, value) {+ Ok(value) => match frame_event(target_format, value, &redactor) {Ok(event) => event,Err(error) => {failed = true;- error_event(target_format, error.to_string())+ error_event(target_format, error.to_string(), &redactor)}},- // The upstream's own error event, already in the target format: forward it- // verbatim so its code and type survive, rather than synthesizing one.+ // Preserve the upstream error's fields, apart from credential redaction,+ // rather than replacing it with a synthesized error.Err(LlmStreamError::Upstream(value)) => {failed = true;- frame_event(target_format, value.clone()).unwrap_or_else(|error| {+ frame_event(target_format, value.clone(), &redactor).unwrap_or_else(|error| {tracing::warn!(error = %error, "in-band error event could not be framed");- error_event(target_format, value.to_string())+ error_event(target_format, value.to_string(), &redactor)})}Err(LlmStreamError::Client(error)) => {tracing::warn!(error = %error, "stream iteration failed");failed = true;- error_event(target_format, error.to_string())+ error_event(target_format, error.to_string(), &redactor)}};yield Ok(event);@@ -62,41 +66,50 @@ pub(crate) fn frame_stream(Sse::new(Box::pin(framed) as SseFrameStream)}-fn frame_event(target_format: WireFormat, value: Value) -> Result<Event, axum::Error> {+fn frame_event(+ target_format: WireFormat,+ value: Value,+ redactor: &Redactor,+) -> Result<Event, serde_json::Error> {+ let data = redactor.json(serde_json::to_string(&value)?);match target_format {- WireFormat::OpenAiChat => Event::default().json_data(value),+ WireFormat::OpenAiChat => Ok(Event::default().data(data)),WireFormat::AnthropicMessages | WireFormat::OpenAiResponses => {let event_type = value.get("type").and_then(Value::as_str).unwrap_or("message").to_string();- Event::default().event(event_type).json_data(value)+ Ok(Event::default().event(event_type).data(data))}}}-fn error_event(target_format: WireFormat, message: String) -> Event {+fn error_event(target_format: WireFormat, message: String, redactor: &Redactor) -> Event {match target_format {WireFormat::OpenAiChat => Event::default().data(- json!({- "error": {- "message": message,- "type": "SwitchyardError",- }- })- .to_string(),- ),- WireFormat::AnthropicMessages | WireFormat::OpenAiResponses => {- Event::default().event("error").data(+ redactor.json(json!({- "type": "error","error": {"message": message,"type": "SwitchyardError",}}).to_string(),+ ),+ ),+ WireFormat::AnthropicMessages | WireFormat::OpenAiResponses => {+ Event::default().event("error").data(+ redactor.json(+ json!({+ "type": "error",+ "error": {+ "message": message,+ "type": "SwitchyardError",+ }+ })+ .to_string(),+ ),)}}@@ -117,7 +130,12 @@ mod tests {// Renders a framed body for one Chat stream.async fn chat_body(items: Vec<Result<Value, LlmStreamError>>) -> TestResult<String> {let stream: RawEventStream = Box::pin(stream::iter(items));- let response = frame_stream(stream, WireFormat::OpenAiChat).into_response();+ let response = frame_stream(+ stream,+ WireFormat::OpenAiChat,+ Arc::new(Redactor::default()),+ )+ .into_response();Ok(String::from_utf8(to_bytes(response.into_body(), usize::MAX).await?.to_vec(),)?)