Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use crate::Result;
use crate::core::algorithm::Driver;
use crate::core::state::State;
use async_trait::async_trait;
use switchyard_protocol::{AggLlmResponse, Category, ModelId, Request};
/// An event observed by the algorithm. Events are consumed by [`Processor`] to mutate state.
///
/// Request-bearing variants ([`Event::Request`], [`Event::Decision`]) borrow the request
/// mutably, so a processor may rewrite it in place and the edit propagates to the rest of
/// the chain and to the model call. The observation-only variants stay immutable.
pub enum Event<'a> {
/// The inbound request that begins a turn.
Request {
/// The request, rewritable in place.
request: &'a mut Request,
/// Offered so a processor can consult a model before the cascade runs.
driver: &'a Driver,
},
/// A routing decision paired with the request that produced it.
///
/// The request is rewritable: a processor may add instructions or notes here that
/// are bound to the routing outcome (e.g. a model-specific system prompt).
Decision {
/// The request, rewritable in place.
request: &'a mut Request,
/// The model selected for `request`.
selected_model_id: &'a ModelId,
/// The category `selected_model_id` was drawn from, when the deciding
/// classifier picked one. `None` for a decision made without a category.
category: Option<Category>,
/// Offered so a processor can inspect the runtime model categories.
driver: &'a Driver,
},
/// A buffered response received back from a model.
ModelResponse(&'a AggLlmResponse),
}
/// Collects events as the algorithm runs and mutates the composition's state.
#[async_trait]
pub trait Processor: Send + Sync {
/// Process an event, accumulating facts into `state`. Request-bearing events
/// ([`Event::Request`], [`Event::Decision`]) may also be rewritten in place.
async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()>;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::state::StateValue;
use crate::core::testing::empty_driver;
use switchyard_protocol::{text_request, text_response};
/// The key each event variant tallies under.
fn event_key(event: &Event<'_>) -> &'static str {
match event {
Event::Request { .. } => "requests",
Event::Decision { .. } => "decisions",
Event::ModelResponse(_) => "model_responses",
}
}
/// Reads a count, treating a missing key as zero.
fn count(state: &State, key: &'static str) -> u32 {
match state.extra.get(key) {
Some(StateValue::Count(n)) => *n,
_ => 0,
}
}
/// Tallies each event variant under its own key.
struct CountingProcessor;
#[async_trait]
impl Processor for CountingProcessor {
async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
let key = event_key(&event);
let next = count(state, key) + 1;
state.extra.insert(key.to_string(), StateValue::Count(next));
Ok(())
}
}
fn request() -> Request {
Request {
llm_request: text_request(Some("auto".to_string()), "hi"),
raw_request: None,
metadata: None,
}
}
#[tokio::test]
async fn processor_tallies_each_event_variant_into_state() -> Result<()> {
let processor = CountingProcessor;
let mut state = State::default();
let mut req = request();
let response = text_response(None, "ok");
let selected_model_id = ModelId::from("test/model");
// Feed one of every event variant through the processor.
processor
.process(
&mut state,
Event::Request {
request: &mut req,
driver: &empty_driver(),
},
)
.await?;
processor
.process(&mut state, Event::ModelResponse(&response))
.await?;
processor
.process(
&mut state,
Event::Decision {
request: &mut req,
selected_model_id: &selected_model_id,
category: None,
driver: &empty_driver(),
},
)
.await?;
assert_eq!(count(&state, "requests"), 1);
assert_eq!(count(&state, "decisions"), 1);
assert_eq!(count(&state, "model_responses"), 1);
Ok(())
}
#[tokio::test]
async fn process_accumulates_state_across_repeated_events() -> Result<()> {
let processor = CountingProcessor;
let mut state = State::default();
let mut req = request();
for _ in 0..3 {
processor
.process(
&mut state,
Event::Request {
request: &mut req,
driver: &empty_driver(),
},
)
.await?;
}
assert_eq!(count(&state, "requests"), 3);
Ok(())
}
/// Rewrites the requested model on every request-bearing event.
struct RewritingProcessor;
#[async_trait]
impl Processor for RewritingProcessor {
async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
match event {
Event::Request { request, .. } | Event::Decision { request, .. } => {
request.llm_request.model = Some("rewritten".to_string());
}
_ => {}
}
Ok(())
}
}
#[tokio::test]
async fn processor_rewrites_the_request_in_place() -> Result<()> {
let mut state = State::default();
let mut req = request();
assert_eq!(req.model_id(), Some("auto".into()));
RewritingProcessor
.process(
&mut state,
Event::Request {
request: &mut req,
driver: &empty_driver(),
},
)
.await?;
// The edit outlives the call, so the next component sees the rewritten request.
assert_eq!(req.model_id(), Some("rewritten".into()));
Ok(())
}
}