Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): enforce runtime target restrictions (#924)

Alex Fournier5cd34f5be107
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 0d0fa88da..b4032c6a4 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -51,6 +51,7 @@ pub struct RuntimeModels {
/// When using subagents this is the parent agent category.
by_category: HashMap<Category, Vec<ModelId>>,
subagent: Option<HashMap<Category, Vec<ModelId>>>,
+ has_target_restriction: bool,
}
impl RuntimeModels {
@@ -59,6 +60,7 @@ impl RuntimeModels {
Self {
by_category,
subagent: None,
+ has_target_restriction: false,
}
}
@@ -68,6 +70,46 @@ impl RuntimeModels {
self
}
+ /// Rejects routing calls and completion candidates outside the configured model groups.
+ /// Enforced by the driver and [`drive`]; hosts bind IDs to permitted deployments.
+ pub fn with_target_restriction(mut self) -> Self {
+ self.has_target_restriction = true;
+ self
+ }
+
+ fn check_targets(&self, targets: &[ModelId]) -> Result<()> {
+ if !self.has_target_restriction {
+ return Ok(());
+ }
+ // Include delegated groups: the parent driver publishes subagent outcomes too.
+ for target in targets {
+ if !self
+ .by_category
+ .values()
+ .chain(self.subagent.iter().flat_map(HashMap::values))
+ .any(|models| models.contains(target))
+ {
+ return Err(LibsyError::TargetNotFound {
+ target: target.clone(),
+ });
+ }
+ }
+ Ok(())
+ }
+
+ fn check_request_targets(&self, request: &Request, targets: &[ModelId]) -> Result<()> {
+ self.check_targets(targets)?;
+ if self.has_target_restriction {
+ let selected = targets.first().ok_or(LibsyError::NoTargets)?;
+ if request.llm_request.model.as_deref() != Some(selected.as_str()) {
+ return Err(LibsyError::AlgorithmError {
+ message: "request model does not match selected target".into(),
+ });
+ }
+ }
+ Ok(())
+ }
+
/// The models in `category`, ordered best-first.
pub fn models_for(&self, category: &Category) -> &[ModelId] {
self.by_category.get(category).map_or(&[], Vec::as_slice)
@@ -473,6 +515,7 @@ impl Driver {
let Some(selected_model_id) = models.first() else {
return Err(LibsyError::NoTargets);
};
+ self.models.check_targets(&models)?;
request.llm_request.model = Some(selected_model_id.to_string());
let started = Instant::now();
let (reply, response) = oneshot::channel::<Result<Response>>();
@@ -517,6 +560,7 @@ impl Driver {
mut request: DecisionRequest,
model: ModelId,
) -> Result<DecisionResponse> {
+ self.models.check_targets(std::slice::from_ref(&model))?;
request.model = Some(model.clone());
let started = Instant::now();
let (reply, response) = oneshot::channel();
@@ -665,7 +709,7 @@ where
F: Fn(Call) -> Fut,
Fut: Future<Output = Result<()>>,
{
- let stream = algorithm.run_stream(request, models, state);
+ let stream = algorithm.run_stream(request, Arc::clone(&models), state);
tokio::pin!(stream);
let mut in_flight = futures::stream::FuturesUnordered::new();
@@ -681,11 +725,25 @@ where
match step {
None => break, // stream has ended, no more steps
Some(item) => match item? {
- Step::CallModel(call) => in_flight.push(serve(Call::Model(call))),
+ Step::CallModel(call) => {
+ models.check_request_targets(&call.request, &call.models)?;
+ in_flight.push(serve(Call::Model(call)));
+ }
Step::CallDecision(call) => {
+ models.check_targets(std::slice::from_ref(&call.model))?;
+ if models.has_target_restriction
+ && call.request.model.as_ref() != Some(&call.model)
+ {
+ return Err(LibsyError::AlgorithmError {
+ message: "decision payload model does not match selected target".into(),
+ });
+ }
in_flight.push(serve(Call::Decision(call)));
}
Step::Done(outcome) => {
+ if outcome.response.is_none() {
+ models.check_request_targets(&outcome.request, &outcome.selected_model_ids)?;
+ }
final_outcome = Some(*outcome);
break;
}
@@ -836,15 +894,21 @@ pub trait Algorithm: Send + Sync + 'static {
let algorithm = self.name().to_string();
// Catch a panicking algorithm so the run still publishes a terminal step.
let route = AssertUnwindSafe(self.route(driver.clone(), request)).catch_unwind();
- let result = observability::observe_run(&algorithm, async move {
- route.await.unwrap_or_else(|payload| {
+ let result = observability::observe_run(&algorithm, async {
+ let outcome = route.await.unwrap_or_else(|payload| {
Err(LibsyError::AlgorithmError {
message: format!(
"algorithm task panicked: {}",
panic_message(payload.as_ref())
),
})
- })
+ })?;
+ if outcome.response.is_none() {
+ driver
+ .models
+ .check_request_targets(&outcome.request, &outcome.selected_model_ids)?;
+ }
+ Ok(outcome)
})
.await;
@@ -999,6 +1063,222 @@ mod tests {
names.iter().map(|name| ModelId::from(*name)).collect()
}
+ enum TargetProbe {
+ Model(&'static [&'static str]),
+ Decision(&'static str),
+ Completion(&'static [&'static str]),
+ MismatchedCompletion,
+ }
+
+ #[async_trait]
+ impl Algorithm for TargetProbe {
+ fn name(&self) -> &str {
+ "target_probe"
+ }
+
+ async fn route(
+ self: Arc<Self>,
+ driver: Driver,
+ request: Request,
+ ) -> Result<RoutingOutcome> {
+ match &*self {
+ Self::Model(models) => {
+ let response = driver
+ .call_model(request.clone(), target_set(models))
+ .await?;
+ Ok(RoutingOutcome::answered(
+ models[0].into(),
+ request,
+ response,
+ ))
+ }
+ Self::Decision(model) => {
+ driver
+ .call_decision(
+ DecisionRequest {
+ model: None,
+ context: Value::Null,
+ questions: Default::default(),
+ },
+ (*model).into(),
+ )
+ .await?;
+ Ok(RoutingOutcome::route_to("allowed".into(), vec![], request))
+ }
+ Self::Completion(models) => Ok(RoutingOutcome::route_to(
+ models[0].into(),
+ target_set(&models[1..]),
+ request,
+ )),
+ Self::MismatchedCompletion => {
+ let mut outcome = RoutingOutcome::route_to("allowed".into(), vec![], request);
+ outcome.request.llm_request.model = Some("outside".into());
+ Ok(outcome)
+ }
+ }
+ }
+ }
+
+ struct RewrittenSteps(Arc<TargetProbe>, fn(&mut Step));
+
+ #[async_trait]
+ impl Algorithm for RewrittenSteps {
+ fn name(&self) -> &str {
+ "rewritten_steps"
+ }
+
+ async fn route(
+ self: Arc<Self>,
+ driver: Driver,
+ request: Request,
+ ) -> Result<RoutingOutcome> {
+ Arc::clone(&self.0).route(driver, request).await
+ }
+
+ fn run_stream(
+ self: Arc<Self>,
+ request: Request,
+ models: Arc<RuntimeModels>,
+ state: State,
+ ) -> StepStream {
+ Box::pin(
+ Arc::clone(&self.0)
+ .run_stream(request, models, state)
+ .map(move |step| {
+ step.map(|mut step| {
+ (self.1)(&mut step);
+ step
+ })
+ }),
+ )
+ }
+ }
+
+ fn restricted_models() -> RuntimeModels {
+ RuntimeModels::new(HashMap::from([
+ (Category::Any, target_set(&["allowed", "fallback"])),
+ (Category::Judge, target_set(&["judge"])),
+ ]))
+ .with_target_restriction()
+ .with_subagent(HashMap::from([(Category::Any, target_set(&["subagent"]))]))
+ }
+
+ #[tokio::test]
+ async fn target_restriction_checks_default_streams() {
+ for probe in [
+ TargetProbe::Model(&["outside"]),
+ TargetProbe::Model(&["allowed", "outside"]),
+ TargetProbe::Decision("outside"),
+ TargetProbe::Completion(&["outside"]),
+ TargetProbe::Completion(&["allowed", "outside"]),
+ ] {
+ let mut stream = Arc::new(probe).run_stream(
+ request(),
+ Arc::new(restricted_models()),
+ State::default(),
+ );
+ assert!(matches!(
+ stream.next().await,
+ Some(Err(LibsyError::TargetNotFound { target })) if target == "outside"
+ ));
+ assert!(stream.next().await.is_none());
+ }
+ let mut stream = Arc::new(TargetProbe::MismatchedCompletion).run_stream(
+ request(),
+ Arc::new(restricted_models()),
+ State::default(),
+ );
+ assert!(matches!(
+ stream.next().await,
+ Some(Err(LibsyError::AlgorithmError { .. }))
+ ));
+
+ let mut stream = Arc::new(TargetProbe::Model(&["outside"])).run_stream(
+ request(),
+ Arc::new(RuntimeModels::default()),
+ State::default(),
+ );
+ assert!(matches!(stream.next().await, Some(Ok(Step::CallModel(_)))));
+ }
+
+ #[tokio::test]
+ async fn target_restriction_rechecks_rewritten_steps() {
+ for probe in [
+ TargetProbe::Model(&["allowed"]),
+ TargetProbe::Decision("judge"),
+ TargetProbe::Completion(&["allowed"]),
+ ] {
+ let probe = Arc::new(probe);
+ let run = |rewrite| {
+ drive(
+ Arc::new(RewrittenSteps(Arc::clone(&probe), rewrite)),
+ request(),
+ Arc::new(restricted_models()),
+ |_| async { panic!("invalid work must not reach the host") },
+ )
+ };
+ let result = run(|step| match step {
+ Step::CallModel(call) => call.models.push("outside".into()),
+ Step::CallDecision(call) => call.model = "outside".into(),
+ Step::Done(outcome) => outcome.selected_model_ids.push("outside".into()),
+ })
+ .await;
+ assert!(
+ matches!(result, Err(LibsyError::TargetNotFound { target }) if target == "outside")
+ );
+
+ let result = run(|step| match step {
+ Step::CallModel(call) => call.request.llm_request.model = Some("outside".into()),
+ Step::CallDecision(call) => call.request.model = Some("allowed".into()),
+ Step::Done(outcome) => outcome.request.llm_request.model = Some("outside".into()),
+ })
+ .await;
+ assert!(matches!(result, Err(LibsyError::AlgorithmError { .. })));
+ }
+ }
+
+ #[tokio::test]
+ async fn target_restriction_preserves_allowed_work() -> Result<()> {
+ for (probe, expected) in [
+ (
+ TargetProbe::Model(&["allowed", "fallback"]),
+ &["allowed"][..],
+ ),
+ (TargetProbe::Model(&["judge"]), &["judge"][..]),
+ (TargetProbe::Decision("judge"), &["allowed"][..]),
+ (
+ TargetProbe::Completion(&["allowed", "fallback"]),
+ &["allowed", "fallback"][..],
+ ),
+ (TargetProbe::Completion(&["subagent"]), &["subagent"][..]),
+ ] {
+ let outcome = drive(
+ Arc::new(probe),
+ request(),
+ Arc::new(restricted_models()),
+ |call| async move {
+ match call {
+ Call::Model(call) => {
+ call.respond(std::future::ready(Ok(reply("ok")))).await
+ }
+ Call::Decision(call) => serve_decision(*call).await,
+ }
+ },
+ )
+ .await?;
+ assert_eq!(outcome.selected_model_ids, target_set(expected));
+ }
+ let outcome = drive(
+ Arc::new(crate::Noop {}),
+ request(),
+ Arc::new(RuntimeModels::default().with_target_restriction()),
+ |_| async { panic!("Noop must not request model calls") },
+ )
+ .await?;
+ assert!(outcome.response.is_some());
+ Ok(())
+ }
+
#[tokio::test]
async fn typed_driver_preserves_call_and_stream_boundaries() -> Result<()> {
tokio::time::timeout(std::time::Duration::from_secs(1), async {