Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: task based llm classifier and judge primitives (#161)

Ayush Agarwalb0e254d38f49
diff --git a/Cargo.lock b/Cargo.lock
index 514bff6a0..1cc8e7946 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1778,6 +1778,7 @@ dependencies = [
"opentelemetry_sdk",
"parking_lot",
"rand 0.8.7",
+ "serde",
"serde_json",
"switchyard-llm-client",
"switchyard-protocol",
diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml
index d2a97903c..c9e58e186 100644
--- a/crates/libsy/Cargo.toml
+++ b/crates/libsy/Cargo.toml
@@ -13,6 +13,7 @@ rust-version.workspace = true
[dependencies]
async-trait.workspace = true
+serde.workspace = true
serde_json.workspace = true
futures.workspace = true
# Metrics-only OTel API: instruments record through the host-installed global
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index 87c3a2aa9..d1ec9081b 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -11,8 +11,8 @@ so it drops into a proxy, gateway, or agent runtime.
Build a target set, pick an algorithm, run a request:
```rust
-use switchyard_libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request};
-use switchyard_libsy::algorithms::LlmClassifier;
+use switchyard_libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request, SharedState};
+use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};
use switchyard_protocol::{completion_text, text_request};
use std::sync::Arc;
@@ -20,10 +20,13 @@ use std::sync::Arc;
let client = Arc::new(MyClient { /* .. */ }) as Arc<dyn RoutedLlmClient>;
let target = |name: &str| LlmTarget { semantic_name: name.into(), llm_client: Some(client.clone()) };
-let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifier::new(
- "classifier", "strong", "weak", 0.5,
- LlmTargetSet::new(vec![target("classifier"), target("strong"), target("weak")]),
-));
+let targets = LlmTargetSet::new(vec![target("classifier"), target("strong"), target("weak")]);
+let judge_target = targets.get_target("classifier")?;
+let algo: Arc<dyn Algorithm<SharedState>> = Arc::new(
+ FallThrough::new(targets).with_classifier(Arc::new(LlmTaskClassifier::new(
+ judge_target, "weak", "strong",
+ )?)),
+);
let req = Request {
// `text_request` is the single-turn shortcut; build an `LlmRequest` directly for
@@ -111,7 +114,8 @@ the id it calls — they can differ (`"strong"` → `"openai/gpt-4o"`) or coinci
## Running a request
-Hold the algorithm as `Arc<dyn Algorithm>` and choose one of two entry points:
+Hold the algorithm as `Arc<dyn Algorithm>` (or `Arc<dyn Algorithm<SharedState>>` for
+`FallThrough`) and choose one of two entry points:
```rust
// run: libsy drives the request to completion, serving each call with the target's
@@ -217,40 +221,12 @@ pub trait Decision: Send + Sync {
}
```
-Give it a `new(config.., target_set)` constructor and `Arc`-wrap it — there is no builder.
-Example — the LLM classifier (classify, then route; full version in
-[`src/algorithms/llm_class.rs`](src/algorithms/llm_class.rs)):
+Compose a classifier into `FallThrough`; the fall-through algorithm calls the judge,
+applies its policy, and then invokes the selected target:
```rust
-#[async_trait]
-impl Algorithm for LlmClassifier {
- fn name(&self) -> &str { "llm_classifier" }
-
- async fn create_run_task(self: Arc<Self>, ctx: Context, driver: Driver, request: Request)
- -> switchyard_libsy::Result<Response> {
- // Thread `ctx` into every offloaded call and decision — it carries the request's
- // cross-cutting state (correlation ids, budgets) for observers downstream.
-
- // 1. Classify: ask the classifier target for a score.
- let classifier = self.target_set.get_target(&self.classifier_model)?;
- driver.info(ctx.clone(), classify_decision.clone()).await?;
- let classify_response =
- driver.call_llm_target(ctx.clone(), &classifier, classify_req, classify_decision).await?;
- // Abbreviated: fold the (buffered or streamed) response to its aggregate and read
- // the completion text as a score. `.as_agg()` alone is `None` for an unfolded stream.
- let score = classify_response.llm_response.into_agg().await
- .ok()
- .and_then(|agg| completion_text(&agg).trim().parse::<f64>().ok());
-
- // 2. Route: strong if score >= threshold, else weak (fail open on None).
- let model = if score.map_or(true, |s| s >= self.threshold) { &self.strong_model } else { &self.weak_model };
- let routed = self.target_set.get_target(model)?;
- driver.info(ctx.clone(), route_decision.clone()).await?;
- driver.call_llm_target(ctx, &routed, routed_req, route_decision).await
- }
-
- async fn process_signals(self: Arc<Self>, _s: Signals) -> switchyard_libsy::Result<()> { Ok(()) }
-}
+let classifier = LlmTaskClassifier::new(judge_target, "weak", "strong")?;
+let router = FallThrough::new(targets).with_classifier(Arc::new(classifier));
```
## Errors
@@ -300,15 +276,15 @@ instrumentation is a no-op.
## Explore
-The core crate includes weighted random routing and naive LLM classifier. Runnable
+The core crate includes weighted random routing and a judge-backed LLM classifier. Runnable
agents live in [`examples`](examples/) folder.
**Reference algorithms** — implementations to read and route with:
- [`Random`](src/algorithms/rand.rs) — uniform or weighted random over the set
(one call).
-- [`LlmClassifier`](src/algorithms/llm_class.rs) — classify, then route
- strong/weak; fail open to strong.
+- [`LlmTaskClassifier`](src/algorithms/llm_class.rs) — capability judge for a
+ [`FallThrough`](src/algorithms/fall_through.rs) classifier cascade.
- [`EnsembleOrchAlgo`](examples/ensemble.rs) — stateful: fan out to
candidates, judge the best, commit to the winner after N exploration turns.
diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rs
index 6ba648515..f9ff0f44d 100644
--- a/crates/libsy/examples/research_agent.rs
+++ b/crates/libsy/examples/research_agent.rs
@@ -6,24 +6,26 @@
//! Every target owns an `RoutedLlmClient`, so the agent runs each request to completion with
//! [`Algorithm::run`]: it serves each offloaded call with the routed
//! target's `default_client` and returns the final response — no stream to drive. The
-//! multi-step routing (classify -> route) happens inside the classifier algorithm; the
-//! agent never sees it. To drive the step stream yourself instead, use
+//! classifier cascade runs inside `FallThrough`; the agent never sees it. To drive the step
+//! stream yourself instead, use
//! `Algorithm::run_stream`. Run with:
//! cargo run -p libsy --example research_agent
use std::sync::Arc;
use async_trait::async_trait;
-use switchyard_libsy::algorithms::LlmClassifier;
+use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};
use switchyard_libsy::{
Algorithm, Context, Decision, LibsyError, LlmResponse, LlmTarget, LlmTargetSet, Request,
- Response, Result, RoutedLlmClient,
+ Response, Result, RoutedLlmClient, SharedState,
};
use switchyard_protocol::{completion_text, text_request, text_response};
const CLASSIFIER: &str = "classifier/model";
const STRONG: &str = "strong/model";
const WEAK: &str = "weak/model";
+/// Lowest judge-estimated solve probability that still routes to the weak model.
+const THRESHOLD: f64 = 0.5;
/// Stub transport. Real integrators implement `RoutedLlmClient` over their own HTTP.
struct StubClient;
@@ -39,9 +41,9 @@ impl RoutedLlmClient for StubClient {
// The model to call is the routed decision's selection, not the inbound name.
let model = decision.selected_model().to_string();
println!(" -> model call: {model}");
- // The classifier returns a score; other models return an answer.
+ // The judge returns a structured verdict; other models return an answer.
let completion = if model == CLASSIFIER {
- "0.9".to_string()
+ r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#.to_string()
} else {
format!("answer from {model}")
};
@@ -62,7 +64,7 @@ fn targets() -> LlmTargetSet {
}
struct ResearchAgent {
- algo: Arc<dyn Algorithm>,
+ algo: Arc<dyn Algorithm<SharedState>>,
}
impl ResearchAgent {
@@ -94,15 +96,17 @@ impl ResearchAgent {
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<()> {
- // Configure routing once: an LLM classifier over three named targets. Swapping
- // in `Random` needs no change to the agent.
- let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifier::new(
- CLASSIFIER,
- STRONG,
- WEAK,
- 0.5,
- targets(),
- )?);
+ let target_set = targets();
+ // Resolving every target up front means an unknown name fails here rather than on the
+ // first request, after the judge call has already been made.
+ let classifier = target_set.get_target(CLASSIFIER)?;
+ let weak = target_set.get_target(WEAK)?;
+ let strong = target_set.get_target(STRONG)?;
+ let algo: Arc<dyn Algorithm<SharedState>> = Arc::new(
+ FallThrough::new(target_set).with_classifier(Arc::new(LlmTaskClassifier::new(
+ classifier, weak, strong, THRESHOLD,
+ )?)),
+ );
let agent = ResearchAgent { algo };
println!("{}", agent.run("what is switchyard?").await?);
diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rs
index b653f4e6e..f0fc11ead 100644
--- a/crates/libsy/examples/research_agent_core.rs
+++ b/crates/libsy/examples/research_agent_core.rs
@@ -6,15 +6,15 @@
//! With no client, every `driver.call_llm_target` is offloaded as a promise the orchestrator
//! surfaces as a `CallLlm` step. The agent makes the "real" model call itself and
//! fulfills the promise — this is the offload/streaming path ("ask, don't call").
-//! The classifier's two steps show up as two `model call:` lines. Run with:
+//! The judge and selected target show up as two `model call:` lines. Run with:
//! cargo run -p libsy --example research_agent_core
use std::sync::Arc;
-use switchyard_libsy::algorithms::LlmClassifier;
+use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};
use switchyard_libsy::{
Algorithm, Context, Decision, LibsyError, LlmResponse, LlmTarget, LlmTargetSet, Request,
- Response, Result, Step,
+ Response, Result, SharedState, Step,
};
use switchyard_protocol::{completion_text, text_request, text_response};
use tokio_stream::StreamExt;
@@ -22,6 +22,8 @@ use tokio_stream::StreamExt;
const CLASSIFIER: &str = "classifier/model";
const STRONG: &str = "strong/model";
const WEAK: &str = "weak/model";
+/// Lowest judge-estimated solve probability that still routes to the weak model.
+const THRESHOLD: f64 = 0.5;
/// The "real" model call the agent makes to fulfill a promise. The core never
/// makes the call itself — it hands back a request and waits for the response.
@@ -29,7 +31,7 @@ const WEAK: &str = "weak/model";
async fn call_model(model: &str) -> Response {
println!(" -> model call: {model}");
let completion = if model == CLASSIFIER {
- "0.9".to_string()
+ r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#.to_string()
} else {
format!("answer from {model}")
};
@@ -49,7 +51,7 @@ fn targets() -> LlmTargetSet {
}
struct ResearchAgent {
- algo: Arc<dyn Algorithm>,
+ algo: Arc<dyn Algorithm<SharedState>>,
}
impl ResearchAgent {
@@ -102,13 +104,17 @@ fn print_decision(decision: &dyn Decision) {
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<()> {
- let algo: Arc<dyn Algorithm> = Arc::new(LlmClassifier::new(
- CLASSIFIER,
- STRONG,
- WEAK,
- 0.5,
- targets(),
- )?);
+ let target_set = targets();
+ // Resolving every target up front means an unknown name fails here rather than on the
+ // first request, after the judge call has already been made.
+ let classifier = target_set.get_target(CLASSIFIER)?;
+ let weak = target_set.get_target(WEAK)?;
+ let strong = target_set.get_target(STRONG)?;
+ let algo: Arc<dyn Algorithm<SharedState>> = Arc::new(
+ FallThrough::new(target_set).with_classifier(Arc::new(LlmTaskClassifier::new(
+ classifier, weak, strong, THRESHOLD,
+ )?)),
+ );
let mut agent = ResearchAgent { algo };
println!("{}", agent.run("what is switchyard?").await?);
diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs
index 1b6538625..422a6865a 100644
--- a/crates/libsy/src/algorithms.rs
+++ b/crates/libsy/src/algorithms.rs
@@ -13,7 +13,7 @@ pub mod passthrough;
pub mod rand;
pub use fall_through::{FallThrough, FallThroughDecision};
-pub use llm_class::{ClassifierDecision, ClassifierTier, LlmClassifier};
+pub use llm_class::LlmTaskClassifier;
pub use noop::{Noop, NoopDecision};
pub use passthrough::{Passthrough, PassthroughDecision};
pub use rand::{Random, RandomDecision};
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 53bb539c4..19e5dfc05 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -1,118 +1,54 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! LLM-classifier router built on the [`Algorithm`] interfaces.
+//! Capability routing with a judge-backed classifier.
//!
-//! Unlike a local ML classifier (which scores a prompt in-process), an LLM
-//! classifier needs its own model call to classify the request. On the new
-//! interfaces this is just two ordinary `driver.call_llm_target`s inside one
-//! `create_run_task`: first the classifier target (to get a score), then the
-//! routed strong/weak target. The multi-step nature is invisible to the caller —
-//! it is the algorithm's own control flow.
-
-use std::sync::Arc;
+//! The classifier judges the full inbound request and emits one decisive target for a
+//! [`FallThrough`](super::FallThrough) cascade. Invalid, abstained, or unavailable judge output
+//! always selects the capable target.
use async_trait::async_trait;
+use serde::Deserialize;
+use serde_json::Value;
+use switchyard_protocol::{LlmRequest, Message, OutputParams, Role};
+use super::util::{Judge, JudgeClassifier, JudgeConfig, JudgePolicy};
use crate::{
- Algorithm, Context, Decision, Driver, LibsyError, LlmTargetSet, Request, Response, Result,
+ Classification, Classifier, Driver, LibsyError, LlmTarget, Request, Result, Score, State,
};
-use switchyard_protocol::{completion_text, LlmRequest, Message, Role};
-
-/// The system prompt tells the classifier what to do
-const CLASSIFIER_SYSTEM_PROMPT: &str = "Rate how strongly this request needs a frontier model. Reply with a single strong-win-rate score in [0, 1].";
-/// The tier a classifier score selected.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum ClassifierTier {
- Strong,
- Weak,
+// TODO: As a first implementation, keeping the prompt and schema paths hardcoded. Add a way to dynamically load and parse user passed prompt and schema.
+const PROMPT_TEMPLATE: &str = include_str!("../prompts/capability-classifier/prompt.md");
+const SCHEMA_TEMPLATE: &str = include_str!("../prompts/capability-classifier/schema.json");
+// TODO: There can be more knobs to tune the classifier after its verdict is parsed. Add more later.
+const RECENT_MESSAGE_WINDOW: usize = 5;
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+/// Parsed judge output. Only confidence, abstention, and solve probability affect v0 routing.
+struct TaskClassifierVerdict {
+ #[serde(rename = "recommended_route")]
+ _recommended_route: String,
+ p_solve: f64,
+ confidence: f64,
+ abstain: bool,
+ #[serde(rename = "capability_boundary")]
+ _capability_boundary: String,
+ #[serde(rename = "primary_rule")]
+ _primary_rule: String,
+ #[serde(rename = "crux")]
+ _crux: String,
}
-impl ClassifierTier {
- /// Stable string form of the tier, used in decision reasoning.
- pub fn as_str(self) -> &'static str {
- match self {
- ClassifierTier::Strong => "strong",
- ClassifierTier::Weak => "weak",
- }
+impl TaskClassifierVerdict {
+ /// Reject out-of-range probabilities before the policy can route efficiently. Range
+ /// containment also rejects NaN and the infinities, which compare false against both bounds.
+ fn is_valid(&self) -> bool {
+ (0.0..=1.0).contains(&self.p_solve) && (0.0..=1.0).contains(&self.confidence)
}
}
-/// Decision produced at each step of the classifier flow. The classify step
-/// leaves `score`/`tier` `None`; the routed step fills them in.
-pub struct ClassifierDecision {
- /// The model this step selected (the classifier model, then the routed model).
- pub selected_model: String,
- /// Human-readable explanation of the step.
- pub reasoning: String,
- /// The classifier score, on the routed step; `None` on the classify step.
- pub score: Option<f64>,
- /// The tier chosen (strong/weak), on the routed step; `None` on the classify step.
- pub tier: Option<ClassifierTier>,
-}
-
-impl Decision for ClassifierDecision {
- fn selected_model(&self) -> &str {
- &self.selected_model
- }
- fn reasoning(&self) -> Option<&str> {
- Some(&self.reasoning)
- }
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-}
-
-/// LLM-classifier router: classify with one target, then route to strong/weak.
-pub struct LlmClassifier {
- classifier_model: String,
- strong_model: String,
- weak_model: String,
- threshold: f64,
- target_set: LlmTargetSet,
-}
-
-impl LlmClassifier {
- /// Configure the classifier: the model that scores each request, the strong
- /// and weak models to route to, the score `threshold` at or above which the
- /// strong model is chosen, and the `target_set` to route among. That set must
- /// contain targets named `classifier_model`, `strong_model`, and `weak_model`.
- ///
- /// # Errors
- ///
- /// Returns an error when `threshold` is outside `0.0..=1.0` or when any
- /// required target name is absent from `target_set`.
- pub fn new(
- classifier_model: impl Into<String>,
- strong_model: impl Into<String>,
- weak_model: impl Into<String>,
- threshold: f64,
- target_set: LlmTargetSet,
- ) -> Result<Self> {
- if !threshold.is_finite() || !(0.0..=1.0).contains(&threshold) {
- return Err(LibsyError::AlgorithmError {
- message: "llm_classifier threshold must be between 0 and 1".to_string(),
- });
- }
- let classifier_model = classifier_model.into();
- let strong_model = strong_model.into();
- let weak_model = weak_model.into();
- target_set.get_target(&classifier_model)?;
- target_set.get_target(&strong_model)?;
- target_set.get_target(&weak_model)?;
- Ok(Self {
- classifier_model,
- strong_model,
- weak_model,
- threshold,
- target_set,
- })
- }
-}
-
-/// The first User message as well as the last <recent_turn_window> turns (User + Assistant).
-/// If fewer than 5 turns have happened, include the whole message.
+/// Keeps client instructions, the initial task, and bounded recent dialogue for the judge.
fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec<Message> {
let mut system = Vec::new();
let mut first_user = None;
@@ -154,414 +90,495 @@ fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec<Message
out
}
-#[async_trait]
-impl Algorithm for LlmClassifier {
- fn name(&self) -> &str {
- "llm_classifier"
- }
+struct CapabilityJudge {
+ config: JudgeConfig,
+}
- async fn create_run_task(
- self: Arc<Self>,
- ctx: Context,
- driver: Driver,
- request: Request,
- ) -> Result<Response> {
- // The agent's inbound name rides through unchanged on every sub-call; the
- // model each sub-call actually hits is carried by its decision instead.
- let inbound = request.llm_request.model.clone();
-
- let mut just_the_key_messages = trim_messages(&request.llm_request.messages, 5);
- just_the_key_messages.insert(0, Message::text(Role::System, CLASSIFIER_SYSTEM_PROMPT));
-
- // 1. Classify: call the classifier target with the score-eliciting prompt.
- let classifier_target = self.target_set.get_target(&self.classifier_model)?;
- let llm_request = LlmRequest {
- model: inbound.clone(),
- messages: just_the_key_messages,
- ..LlmRequest::default()
- };
- let classify_request = Request {
- llm_request,
- raw_request: request.raw_request.clone(),
+impl Judge for CapabilityJudge {
+ type Verdict = TaskClassifierVerdict;
+
+ fn build_request(&self, _state: &State, request: &Request) -> Request {
+ // The judge owns the leading system prompt; client instructions and task context follow.
+ let mut messages = trim_messages(&request.llm_request.messages, RECENT_MESSAGE_WINDOW);
+ messages.insert(
+ 0,
+ Message::text(Role::System, self.config.system_prompt.clone()),
+ );
+ Request {
+ llm_request: LlmRequest {
+ model: request.llm_request.model.clone(),
+ messages,
+ output: OutputParams {
+ response_format: self.config.response_schema.clone(),
+ ..OutputParams::default()
+ },
+ ..LlmRequest::default()
+ },
+ raw_request: None,
metadata: request.metadata.clone(),
+ }
+ }
+}
+
+struct TaskClassifierPolicy {
+ efficient_target: String,
+ capable_target: String,
+ /// Lowest `p_solve` that still routes to the efficient target.
+ threshold: f64,
+}
+
+impl TaskClassifierPolicy {
+ fn new(
+ efficient_target: impl Into<String>,
+ capable_target: impl Into<String>,
+ threshold: f64,
+ ) -> Self {
+ Self {
+ efficient_target: efficient_target.into(),
+ capable_target: capable_target.into(),
+ threshold,
+ }
+ }
+}
+
+impl JudgePolicy for TaskClassifierPolicy {
+ type Verdict = TaskClassifierVerdict;
+
+ fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification {
+ // Judge output is untrusted. Anything incomplete, invalid, abstained, or below the
+ // threshold routes to the capable target rather than risking an underpowered route.
+ let target = match verdict {
+ Some(verdict)
+ if verdict.is_valid() && !verdict.abstain && verdict.p_solve >= self.threshold =>
+ {
+ &self.efficient_target
+ }
+ _ => &self.capable_target,
};
- let classify_decision: Arc<dyn Decision> = Arc::new(ClassifierDecision {
- selected_model: self.classifier_model.clone(),
- reasoning: format!("classifying request via {}", self.classifier_model),
- score: None,
- tier: None,
- });
- driver.info(ctx.clone(), classify_decision.clone()).await?;
- let classify_response = driver
- .call_llm_target(
- ctx.clone(),
- &classifier_target,
- classify_request,
- classify_decision,
- )
- .await?;
- // Drain a streamed classifier response to its aggregate so a streamed
- // score is read instead of silently dropped. Keep only a valid
- // probability in [0.0, 1.0]: NaN, ±inf, and out-of-range values parse
- // as f64 but are not usable scores, so they collapse to None and fail
- // open to strong below rather than being treated as a real verdict.
- let classify_agg = classify_response
- .llm_response
- .into_agg()
- .await
- .map_err(|error| LibsyError::client_call(&self.classifier_model, error))?;
- let score = completion_text(&classify_agg)
- .trim()
- .parse::<f64>()
- .ok()
- .filter(|s| (0.0..=1.0).contains(s));
-
- // 2. Route: pick strong/weak. Fail open — an unparseable or out-of-range
- // score routes strong.
- let (tier, model) = match score {
- Some(s) if s >= self.threshold => (ClassifierTier::Strong, self.strong_model.clone()),
- Some(_) => (ClassifierTier::Weak, self.weak_model.clone()),
- None => (ClassifierTier::Strong, self.strong_model.clone()),
+ Classification::Scores(vec![Score {
+ target: target.clone(),
+ confidence: 1.0,
+ }])
+ }
+}
+
+/// A full-request capability classifier configured with the packaged prompt and schema.
+pub struct LlmTaskClassifier {
+ classifier: JudgeClassifier<CapabilityJudge, TaskClassifierPolicy>,
+}
+
+impl LlmTaskClassifier {
+ /// Selects `efficient_target` when the judge's `p_solve` reaches `threshold`, and
+ /// `capable_target` otherwise. Errors if `threshold` is outside `[0.0, 1.0]`.
+ pub fn new(
+ judge_target: LlmTarget,
+ efficient_target: LlmTarget,
+ capable_target: LlmTarget,
+ threshold: f64,
+ ) -> Result<Self> {
+ if !(0.0..=1.0).contains(&threshold) {
+ return Err(LibsyError::AlgorithmError {
+ message: format!("threshold must be between 0 and 1, got {threshold}"),
+ });
+ }
+ let judge = CapabilityJudge {
+ config: Self::load_judge_config()?,
};
- let routed_target = self.target_set.get_target(&model)?;
- let route_decision: Arc<dyn Decision> = Arc::new(ClassifierDecision {
- reasoning: format!(
- "classifier score {score:?} vs threshold {}; selected {model} ({})",
- self.threshold,
- tier.as_str()
+ Ok(Self {
+ classifier: JudgeClassifier::new(
+ judge,
+ judge_target,
+ TaskClassifierPolicy::new(
+ efficient_target.semantic_name,
+ capable_target.semantic_name,
+ threshold,
+ ),
),
- selected_model: model.clone(),
- score,
- tier: Some(tier),
- });
- driver.info(ctx.clone(), route_decision.clone()).await?;
- driver
- .call_llm_target(ctx, &routed_target, request, route_decision)
- .await
+ })
+ }
+
+ /// Loads the judge configuration from the packaged prompt and schema.
+ /// TODO: Move towards more generic loading and parsing of config when we multiple prompts and schemas to handle for same algorithm
+ fn load_judge_config() -> Result<JudgeConfig> {
+ // The response schema is rendered into the prompt and sent as structured-output metadata.
+ // One asset therefore defines both the instruction contract and provider enforcement.
+ let response_schema: Value =
+ serde_json::from_str(SCHEMA_TEMPLATE).map_err(|error| LibsyError::AlgorithmError {
+ message: format!("capability response schema is invalid: {error}"),
+ })?;
+ let prompt_schema = response_schema
+ .pointer("/json_schema/schema")
+ .ok_or_else(|| LibsyError::AlgorithmError {
+ message: "capability response schema has no json_schema.schema".to_string(),
+ })?;
+ let prompt_schema = serde_json::to_string_pretty(prompt_schema).map_err(|error| {
+ LibsyError::AlgorithmError {
+ message: format!("capability prompt schema could not be rendered: {error}"),
+ }
+ })?;
+ Ok(JudgeConfig {
+ system_prompt: PROMPT_TEMPLATE.replace("{{RESPONSE_SCHEMA}}", &prompt_schema),
+ response_schema: Some(response_schema),
+ })
+ }
+}
+
+#[async_trait]
+impl Classifier for LlmTaskClassifier {
+ async fn score(
+ &self,
+ state: &mut State,
+ request: &Request,
+ driver: Option<&Driver>,
+ ) -> Result<Classification> {
+ self.classifier.score(state, request, driver).await
}
}
#[cfg(test)]
mod tests {
- use super::*;
- use crate::{LlmResponse, LlmResponseChunk, LlmTarget, Response, RoutedLlmClient};
- use futures::StreamExt;
+ use std::sync::Arc;
+
use parking_lot::Mutex;
- use switchyard_protocol::text_response;
-
- /// Returns `score` for the classifier target, an answer tagged with the model
- /// otherwise; records the requests it saw so a test can inspect the classifier
- /// prompt.
- struct ScoringClient {
- classifier_model: String,
- score: String,
- /// When true, the classifier target returns its score as a live stream
- /// rather than a buffered aggregate, exercising the drain-the-stream path.
- stream: bool,
- seen: Arc<Mutex<Vec<Request>>>,
+
+ use super::*;
+ use switchyard_protocol::{completion_text, text_response, LlmClientError};
+
+ use crate::{
+ Algorithm, Context, LlmResponse, LlmTargetSet, Response, RoutedLlmClient, SharedState,
+ };
+
+ const TEST_THRESHOLD: f64 = 0.5;
+
+ fn policy() -> TaskClassifierPolicy {
+ TaskClassifierPolicy::new("efficient", "capable", TEST_THRESHOLD)
+ }
+
+ /// A verdict whose non-routing fields are fixed — only the three the policy reads vary.
+ fn verdict(p_solve: f64, confidence: f64, abstain: bool) -> TaskClassifierVerdict {
+ TaskClassifierVerdict {
+ _recommended_route: "efficient".to_string(),
+ p_solve,
+ confidence,
+ abstain,
+ _capability_boundary: "supported".to_string(),
+ _primary_rule: "SUP-1".to_string(),
+ _crux: "test crux".to_string(),
+ }
+ }
+
+ fn selected(
+ policy: &TaskClassifierPolicy,
+ verdict: Option<&TaskClassifierVerdict>,
+ ) -> Result<String> {
+ policy
+ .to_classification(verdict)
+ .argmax(false)?
+ .map(|score| score.target)
+ .ok_or_else(|| LibsyError::AlgorithmError {
+ message: "policy abstained".to_string(),
+ })
+ }
+
+ #[derive(Default)]
+ struct PerRequestClient {
+ calls: Mutex<Vec<String>>,
+ }
+
+ impl PerRequestClient {
+ fn calls(&self) -> Vec<String> {
+ self.calls.lock().clone()
+ }
}
#[async_trait]
- impl RoutedLlmClient for ScoringClient {
+ impl RoutedLlmClient for PerRequestClient {
async fn call(
&self,
_ctx: Context,
request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- let name = decision.selected_model().to_string();
- let is_classifier = name == self.classifier_model;
- let completion = if is_classifier {
- self.score.clone()
+ decision: Arc<dyn crate::Decision>,
+ ) -> std::result::Result<Response, LlmClientError> {
+ let model = decision.selected_model().to_string();
+ self.calls.lock().push(model.clone());
+ let completion = if model == "judge" {
+ r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#.to_string()
} else {
- format!("answer from {name}")
- };
- self.seen.lock().push(request);
- // A streaming classifier target emits its score as a live TextDelta
- // stream; the algorithm must drain it (into_agg) to read the score.
- let llm_response = if is_classifier && self.stream {
- let chunks = vec![Ok(LlmResponseChunk::TextDelta {
- index: 0,
- text: completion,
- })];
- LlmResponse::Stream(futures::stream::iter(chunks).boxed())
- } else {
- LlmResponse::Agg(text_response(None, completion))
+ format!("answer from {model}")
};
Ok(Response {
- llm_response,
- metadata: None,
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
+ metadata: request.metadata,
})
}
}
- /// Build a classifier algo whose three targets share a scoring client.
- fn algo(threshold: f64, score: &str) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {
- build_algo(threshold, score, false)
- }
+ struct UnreachableJudgeClient;
- /// Like [`algo`], but the classifier target returns its score as a live stream.
- fn algo_streaming(
- threshold: f64,
- score: &str,
- ) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {
- build_algo(threshold, score, true)
+ #[async_trait]
+ impl RoutedLlmClient for UnreachableJudgeClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ request: Request,
+ decision: Arc<dyn crate::Decision>,
+ ) -> std::result::Result<Response, LlmClientError> {
+ let model = decision.selected_model().to_string();
+ if model == "judge" {
+ return Err(LlmClientError::Timeout {
+ source: Box::new(std::io::Error::other("judge unreachable")),
+ });
+ }
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, format!("answer from {model}"))),
+ metadata: request.metadata,
+ })
+ }
}
- fn build_algo(
- threshold: f64,
- score: &str,
- stream: bool,
- ) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {
- let seen = Arc::new(Mutex::new(Vec::new()));
- let client = Arc::new(ScoringClient {
- classifier_model: "router/classifier".to_string(),
- score: score.to_string(),
- stream,
- seen: Arc::clone(&seen),
- }) as Arc<dyn RoutedLlmClient>;
+ fn router(client: Arc<dyn RoutedLlmClient>) -> Result<Arc<super::super::FallThrough>> {
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
llm_client: Some(client.clone()),
};
- let target_set = LlmTargetSet::new(vec![
- target("router/classifier"),
- target("frontier/model"),
- target("cheap/model"),
- ]);
- let algo = LlmClassifier::new(
- "router/classifier",
- "frontier/model",
- "cheap/model",
- threshold,
- target_set,
- )?;
- Ok((algo, seen))
+ let targets = LlmTargetSet::new(vec![target("efficient"), target("capable")]);
+ let efficient = targets.get_target("efficient")?;
+ let capable = targets.get_target("capable")?;
+ Ok(Arc::new(
+ super::super::FallThrough::new(targets).with_classifier(Arc::new(
+ LlmTaskClassifier::new(target("judge"), efficient, capable, TEST_THRESHOLD)?,
+ )),
+ ))
}
- fn request(prompt: &str) -> Request {
+ fn classify_request() -> Request {
Request {
- llm_request: switchyard_protocol::text_request(Some("auto".to_string()), prompt),
+ llm_request: switchyard_protocol::text_request(
+ Some("auto".to_string()),
+ "classify this task",
+ ),
raw_request: None,
metadata: None,
}
}
- /// Wrap a classifier algo as `Arc<dyn Algorithm>` we can drive to completion.
- fn orch(algo: LlmClassifier) -> Arc<dyn Algorithm> {
- Arc::new(algo)
- }
-
- /// Downcast a trace entry to the concrete classifier decision.
- fn as_classifier(d: &Arc<dyn Decision>) -> Result<&ClassifierDecision> {
- d.as_any()
- .downcast_ref::<ClassifierDecision>()
- .ok_or_else(|| {
- crate::DriverError::TypeMismatch {
- expected: "ClassifierDecision",
- }
- .into()
- })
- }
-
#[tokio::test]
- async fn score_at_or_above_threshold_routes_strong() -> Result<()> {
- let (algo, _) = algo(0.5, "0.9")?;
- let (trace, response) = orch(algo)
- .run(Context::default(), request("solve this proof"))
+ async fn an_unreachable_judge_routes_capable_instead_of_failing_the_request() -> Result<()> {
+ let router = router(Arc::new(UnreachableJudgeClient))?;
+
+ let (trace, response) = router
+ .run(Context::<SharedState>::default(), classify_request())
.await?;
+
+ assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
assert_eq!(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- "answer from frontier/model"
+ response.llm_response.as_agg().map(completion_text),
+ Some("answer from capable".to_string())
);
- // Trace: [classify, route].
- assert_eq!(trace[0].selected_model(), "router/classifier");
- let routed = as_classifier(&trace[1])?;
- assert_eq!(routed.selected_model, "frontier/model");
- assert_eq!(routed.tier, Some(ClassifierTier::Strong));
- assert_eq!(routed.score, Some(0.9));
Ok(())
}
#[tokio::test]
- async fn score_below_threshold_routes_weak() -> Result<()> {
- let (algo, _) = algo(0.5, "0.2")?;
- let (trace, response) = orch(algo)
- .run(Context::default(), request("say hello"))
+ async fn classifier_judges_each_request_without_affinity() -> Result<()> {
+ let client = Arc::new(PerRequestClient::default());
+ let router = router(client.clone())?;
+ let request = classify_request;
+
+ router
+ .clone()
+ .run(Context::<SharedState>::default(), request())
.await?;
+ router
+ .clone()
+ .run(Context::<SharedState>::default(), request())
+ .await?;
+
assert_eq!(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- "answer from cheap/model"
+ client.calls(),
+ vec!["judge", "efficient", "judge", "efficient"]
);
- let routed = as_classifier(&trace[1])?;
- assert_eq!(routed.tier, Some(ClassifierTier::Weak));
- assert_eq!(routed.score, Some(0.2));
Ok(())
}
- #[tokio::test]
- async fn score_exactly_at_threshold_routes_strong() -> Result<()> {
- let (algo, _) = algo(0.5, "0.5")?;
- let (_, response) = orch(algo)
- .run(Context::default(), request("borderline"))
- .await?;
- assert_eq!(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- "answer from frontier/model"
- );
+ #[test]
+ fn the_threshold_boundary_is_inclusive() -> Result<()> {
+ let policy = policy();
+ let at_threshold = verdict(0.5, 0.0, false);
+ let below_threshold = verdict(0.49, 1.0, false);
+ assert_eq!(selected(&policy, Some(&at_threshold))?, "efficient");
+ assert_eq!(selected(&policy, Some(&below_threshold))?, "capable");
Ok(())
}
- #[tokio::test]
- async fn unparseable_score_defaults_to_strong() -> Result<()> {
- let (algo, _) = algo(0.5, "not-a-number")?;
- let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
- assert_eq!(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- "answer from frontier/model"
- );
- let routed = as_classifier(&trace[1])?;
- assert_eq!(routed.tier, Some(ClassifierTier::Strong));
- assert_eq!(routed.score, None);
+ #[test]
+ fn the_threshold_moves_the_routing_boundary() -> Result<()> {
+ let borderline = verdict(0.5, 1.0, false);
+ let strict = TaskClassifierPolicy::new("efficient", "capable", 0.9);
+ let lenient = TaskClassifierPolicy::new("efficient", "capable", 0.1);
+ assert_eq!(selected(&strict, Some(&borderline))?, "capable");
+ assert_eq!(selected(&lenient, Some(&borderline))?, "efficient");
Ok(())
}
#[test]
- fn rejects_invalid_threshold() {
- let error = algo(1.5, "0.5")
- .err()
- .map(|error| error.to_string())
- .unwrap_or_default();
- assert!(error.contains("threshold must be between 0 and 1"));
+ fn an_out_of_range_threshold_is_rejected() -> Result<()> {
+ let target = |name: &str| LlmTarget {
+ semantic_name: name.to_string(),
+ llm_client: None,
+ };
+ for bad in [1.5, -0.1, f64::NAN, f64::INFINITY] {
+ assert!(
+ LlmTaskClassifier::new(target("judge"), target("e"), target("c"), bad).is_err(),
+ "threshold {bad} should be rejected"
+ );
+ }
+ LlmTaskClassifier::new(target("judge"), target("e"), target("c"), 0.0)?;
+ LlmTaskClassifier::new(target("judge"), target("e"), target("c"), 1.0)?;
+ Ok(())
}
- #[tokio::test]
- async fn out_of_range_score_defaults_to_strong() -> Result<()> {
- // "NaN" parses as an f64 but is not a usable probability, so it must fail
- // open to strong rather than be treated as a below-threshold verdict and
- // silently downgraded to weak.
- let (algo, _) = algo(0.5, "NaN")?;
- let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
- assert_eq!(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- "answer from frontier/model"
- );
- let routed = as_classifier(&trace[1])?;
- assert_eq!(routed.tier, Some(ClassifierTier::Strong));
- assert_eq!(routed.score, None);
+ #[test]
+ fn invalid_or_abstained_verdict_routes_capable() -> Result<()> {
+ let policy = policy();
+ let invalid_probability = verdict(1.1, 1.0, false);
+ let abstained = verdict(1.0, 1.0, true);
+ assert_eq!(selected(&policy, Some(&invalid_probability))?, "capable");
+ assert_eq!(selected(&policy, Some(&abstained))?, "capable");
+ assert_eq!(selected(&policy, None)?, "capable");
Ok(())
}
- #[tokio::test]
- async fn below_range_score_defaults_to_strong() -> Result<()> {
- // "-0.1" parses as an f64 but is below the valid probability range, so it
- // must fail open to strong. The old code compared it against the threshold
- // (-0.1 < 0.5) and routed weak.
- let (algo, _) = algo(0.5, "-0.1")?;
- let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
+ #[test]
+ fn capability_judge_builds_a_structured_request() -> Result<()> {
+ let judge = CapabilityJudge {
+ config: LlmTaskClassifier::load_judge_config()?,
+ };
+ let request = Request {
+ llm_request: LlmRequest {
+ model: Some("inbound".to_string()),
+ messages: vec![
+ Message::text(Role::System, "client instructions"),
+ Message::text(Role::Developer, "client developer instructions"),
+ Message::text(Role::User, "initial task"),
+ Message::text(Role::Assistant, "old response"),
+ Message::text(Role::User, "old follow-up"),
+ Message::text(Role::Assistant, "recent 1"),
+ Message::text(Role::User, "recent 2"),
+ Message::text(Role::Assistant, "recent 3"),
+ Message::text(Role::User, "recent 4"),
+ Message::text(Role::Assistant, "recent 5"),
+ ],
+ ..LlmRequest::default()
+ },
+ raw_request: None,
+ metadata: None,
+ };
+ let judge_request = judge.build_request(&State::default(), &request);
+
+ assert_eq!(judge_request.llm_request.model, request.llm_request.model);
assert_eq!(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- "answer from frontier/model"
+ judge_request.llm_request.messages.len(),
+ RECENT_MESSAGE_WINDOW + 4
);
- let routed = as_classifier(&trace[1])?;
- assert_eq!(routed.tier, Some(ClassifierTier::Strong));
- assert_eq!(routed.score, None);
- Ok(())
- }
-
- #[tokio::test]
- async fn streamed_score_below_threshold_routes_weak() -> Result<()> {
- // A streaming classifier target must have its score drained and read, not
- // dropped — otherwise every streamed verdict falls open to strong
- // regardless of value.
- let (algo, _) = algo_streaming(0.5, "0.2")?;
- let (trace, response) = orch(algo)
- .run(Context::default(), request("say hello"))
- .await?;
+ let contents = judge_request
+ .llm_request
+ .messages
+ .iter()
+ .filter_map(|message| message.text_content("\n"))
+ .collect::<Vec<_>>();
+ assert!(contents.contains(&"initial task".to_string()));
+ assert!(contents.contains(&"recent 1".to_string()));
+ assert!(contents.contains(&"recent 5".to_string()));
+ assert!(contents.contains(&"client instructions".to_string()));
+ assert!(contents.contains(&"client developer instructions".to_string()));
+ assert!(!contents.contains(&"old response".to_string()));
+ assert!(!contents.contains(&"old follow-up".to_string()));
assert_eq!(
- response
- .llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default(),
- "answer from cheap/model"
+ judge_request.llm_request.output.response_format,
+ judge.config.response_schema
);
- let routed = as_classifier(&trace[1])?;
- assert_eq!(routed.tier, Some(ClassifierTier::Weak));
- assert_eq!(routed.score, Some(0.2));
Ok(())
}
- #[tokio::test]
- async fn classifier_stream_error_propagates_not_swallowed() -> Result<()> {
- // A classifier stream that fails mid-drain must surface as an error, not be
- // swallowed into a None score that silently routes strong. Guards the
- // into_agg() error branch: swapping the `?` for `.ok()` would fail open on a
- // real stream failure, and this test would catch it.
- struct ErroringStreamClient;
- #[async_trait]
- impl RoutedLlmClient for ErroringStreamClient {
- async fn call(
- &self,
- _ctx: Context,
- _request: Request,
- _decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- let chunks = vec![Err(switchyard_protocol::LlmClientError::General(
- "classifier stream failed".to_string(),
- ))];
- Ok(Response {
- llm_response: LlmResponse::Stream(futures::stream::iter(chunks).boxed()),
- metadata: None,
- })
- }
+ fn sample_value(spec: &Value) -> Value {
+ if let Some(first) = spec
+ .get("enum")
+ .and_then(Value::as_array)
+ .and_then(|values| values.first())
+ {
+ return first.clone();
}
- let client = Arc::new(ErroringStreamClient) as Arc<dyn RoutedLlmClient>;
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- llm_client: Some(client.clone()),
- };
- let target_set = LlmTargetSet::new(vec![
- target("router/classifier"),
- target("frontier/model"),
- target("cheap/model"),
- ]);
- let algo = LlmClassifier::new(
- "router/classifier",
- "frontier/model",
- "cheap/model",
- 0.5,
- target_set,
- )?;
- match orch(algo).run(Context::default(), request("hi")).await {
- Err(LibsyError::ClientCall { target, .. }) => assert_eq!(target, "router/classifier"),
- Err(other) => panic!("expected a ClientCall error, got: {other}"),
- Ok(_) => panic!("expected the classifier stream error to propagate, not fail open"),
+ match spec.get("type").and_then(Value::as_str) {
+ Some("number") => serde_json::json!(0.5),
+ Some("boolean") => serde_json::json!(false),
+ _ => serde_json::json!("sample"),
}
+ }
+
+ fn schema_shaped_verdict(schema: &Value) -> Result<String> {
+ let properties = schema
+ .pointer("/json_schema/schema/properties")
+ .and_then(Value::as_object)
+ .ok_or_else(|| LibsyError::AlgorithmError {
+ message: "packaged schema declares no properties".to_string(),
+ })?;
+ Ok(Value::Object(
+ properties
+ .iter()
+ .map(|(name, spec)| (name.clone(), sample_value(spec)))
+ .collect(),
+ )
+ .to_string())
+ }
+
+ /// Built from the schema so a property added there fails here rather than silently
+ /// rejecting every production verdict.
+ #[test]
+ fn every_schema_property_round_trips_through_the_judge_parser() -> Result<()> {
+ let config = LlmTaskClassifier::load_judge_config()?;
+ let schema = config
+ .response_schema
+ .as_ref()
+ .ok_or_else(|| LibsyError::AlgorithmError {
+ message: "packaged judge config has no response schema".to_string(),
+ })?;
+ let reply = schema_shaped_verdict(schema)?;
+ let judge = CapabilityJudge {
+ config: config.clone(),
+ };
+
+ let verdict = judge.parse(&text_response(None, reply))?;
+
+ assert!(verdict.is_valid());
+ assert!(!verdict.abstain);
+ Ok(())
+ }
+
+ #[test]
+ fn prompt_includes_concrete_rules_and_schema() -> Result<()> {
+ let config = LlmTaskClassifier::load_judge_config()?;
+ let prompt = config.system_prompt;
+ assert!(prompt.contains("SUP-1 [supported]"));
+ assert!(!prompt.contains("{{CAPABILITY_RULES}}"));
+ assert!(!prompt.contains("{{PRIMARY_RULE_VALUES}}"));
+ assert!(!prompt.contains("{{RESPONSE_SCHEMA}}"));
+ assert!(prompt.contains("\"type\": \"object\""));
+ assert!(!prompt.contains("\"json_schema\""));
+ assert!(!prompt.contains("\"CapabilityClassifierDecision\""));
+ let rule_values = config
+ .response_schema
+ .as_ref()
+ .and_then(|schema| schema.pointer("/json_schema/schema/properties/primary_rule/enum"))
+ .and_then(Value::as_array)
+ .ok_or_else(|| LibsyError::AlgorithmError {
+ message: "rendered response schema has no primary rule enum".to_string(),
+ })?;
+ assert!(rule_values
+ .iter()
+ .any(|value| value.as_str() == Some("SUP-1")));
+ assert!(rule_values
+ .iter()
+ .any(|value| value.as_str() == Some("none")));
Ok(())
}
}
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index 389a1e72a..9b86746bd 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -34,14 +34,17 @@ impl Decision for NoopDecision {
}
#[async_trait::async_trait]
-impl Algorithm for Noop {
+impl<S> Algorithm<S> for Noop
+where
+ S: Clone + Send + Sync + 'static,
+{
fn name(&self) -> &str {
"noop"
}
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
+ ctx: Context<S>,
driver: Driver,
request: Request,
) -> Result<Response> {
@@ -52,7 +55,7 @@ impl Algorithm for Noop {
let decision: Arc<dyn Decision> = Arc::new(NoopDecision {
model: model.clone(),
});
- driver.info(ctx, decision.clone()).await?;
+ driver.info(ctx.without_state(), decision.clone()).await?;
let llm_response = LlmResponse::Agg(AggLlmResponse {
id: Some("switchyard-noop".to_string()),
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index 96e45ddab..e2b71c100 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -40,20 +40,24 @@ impl Decision for PassthroughDecision {
}
#[async_trait::async_trait]
-impl Algorithm for Passthrough {
+impl<S> Algorithm<S> for Passthrough
+where
+ S: Clone + Send + Sync + 'static,
+{
fn name(&self) -> &str {
"passthrough"
}
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
+ ctx: Context<S>,
driver: Driver,
request: Request,
) -> Result<Response> {
let decision: Arc<dyn Decision> = Arc::new(PassthroughDecision {
model_id: self.target.semantic_name.clone(),
});
+ let ctx = ctx.without_state();
driver.info(ctx.clone(), decision.clone()).await?;
driver
.call_llm_target(ctx, &self.target, request, decision)
@@ -97,7 +101,7 @@ mod tests {
raw_request: None,
metadata: None,
};
- let algorithm = Arc::new(super::Passthrough::new(LlmTarget {
+ let algorithm: Arc<dyn Algorithm> = Arc::new(super::Passthrough::new(LlmTarget {
semantic_name: MODEL_ID.to_string(),
llm_client: Some(Arc::new(EchoClient)),
}));
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 88de59e9d..c91f7cfcf 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -130,14 +130,17 @@ fn invalid_weights(message: String) -> LibsyError {
}
#[async_trait]
-impl Algorithm for Random {
+impl<S> Algorithm<S> for Random
+where
+ S: Clone + Send + Sync + 'static,
+{
fn name(&self) -> &str {
"random"
}
async fn create_run_task(
self: Arc<Self>,
- ctx: Context,
+ ctx: Context<S>,
driver: Driver,
request: Request,
) -> Result<Response> {
@@ -149,6 +152,7 @@ impl Algorithm for Random {
selected_model: selected,
});
+ let ctx = ctx.without_state();
driver.info(ctx.clone(), Arc::clone(&decision)).await?;
driver
.call_llm_target(ctx, &target, request, decision)
@@ -346,9 +350,8 @@ mod tests {
#[tokio::test]
async fn process_signals_is_a_noop() -> Result<()> {
- Arc::new(algorithm(&["only/model"], None, None)?)
- .process_signals(Signals {})
- .await?;
+ let algorithm: Arc<dyn Algorithm> = Arc::new(algorithm(&["only/model"], None, None)?);
+ algorithm.process_signals(Signals {}).await?;
Ok(())
}
diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs
index c77354a8f..827c233c3 100644
--- a/crates/libsy/src/algorithms/util.rs
+++ b/crates/libsy/src/algorithms/util.rs
@@ -3,9 +3,11 @@
mod affinity;
pub(crate) mod handoff_notes;
+mod llm_judge;
pub(crate) mod stage_router;
mod subagent;
pub(crate) mod tool_signals;
pub use affinity::AffinityRouter;
+pub(crate) use llm_judge::{Judge, JudgeClassifier, JudgeConfig, JudgePolicy};
pub use subagent::SubagentOverride;
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
new file mode 100644
index 000000000..8c2ff481b
--- /dev/null
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -0,0 +1,401 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Shared LLM judge primitives.
+//!
+//! [`Judge`] owns algorithm-specific request construction and verdict parsing.
+//! [`JudgeClassifier`] owns the judge model call and hands its verdict to a policy that chooses
+//! the route.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use serde::de::DeserializeOwned;
+use serde_json::Value;
+use switchyard_protocol::{completion_text, AggLlmResponse};
+
+use crate::{
+ Classification, Classifier, Context, Decision, Driver, LibsyError, LlmTarget, Request, Result,
+ State,
+};
+
+#[derive(Clone, Debug, Default)]
+/// Prompt and structured-output contract for one judge.
+pub struct JudgeConfig {
+ /// Prepended instructions that define what the judge evaluates.
+ pub system_prompt: String,
+ /// Optional provider response format that constrains the judge verdict.
+ pub response_schema: Option<Value>,
+}
+
+/// Builds and parses requests for one algorithm-specific LLM judge.
+pub trait Judge: Send + Sync {
+ type Verdict: DeserializeOwned + Send + Sync;
+
+ fn build_request(&self, state: &State, request: &Request) -> Request;
+
+ fn parse(&self, response: &AggLlmResponse) -> Result<Self::Verdict> {
+ parse_json_verdict(response)
+ }
+}
+
+/// Converts a parsed verdict, or an unavailable verdict, into a routing classification.
+/// Consider this as a deterministic policy which can act on the signals predicted from the classifier
+/// and choose the route based on the verdict.
+pub trait JudgePolicy: Send + Sync {
+ type Verdict: Send + Sync;
+
+ fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification;
+}
+
+/// A classifier that calls one judge target and routes through its verdict policy.
+pub struct JudgeClassifier<J, P> {
+ judge: J,
+ target: LlmTarget,
+ policy: P,
+}
+
+impl<J, P> JudgeClassifier<J, P>
+where
+ J: Judge,
+ P: JudgePolicy<Verdict = J::Verdict>,
+{
+ /// Combines a judge target with a verdict policy.
+ pub fn new(judge: J, target: LlmTarget, policy: P) -> Self {
+ Self {
+ judge,
+ target,
+ policy,
+ }
+ }
+
+ /// Consults the judge, yielding `None` when it is unavailable or unintelligible.
+ ///
+ /// A judge is an optimization, not a dependency: failing the caller's request because the
+ /// judge is down would be worse than routing without it, so every failure — transport,
+ /// mid-stream, or unparseable reply — is logged and folded into `None` for the policy's
+ /// fail-closed branch. A closed driver stream is folded too; the algorithm's next driver
+ /// call surfaces it, so nothing is masked.
+ async fn verdict(
+ &self,
+ state: &mut State,
+ request: &Request,
+ driver: &Driver,
+ ) -> Option<J::Verdict> {
+ let judge_model = self.target.semantic_name.as_str();
+ let warn = |error: &dyn std::fmt::Display| {
+ tracing::warn!(
+ target: "libsy",
+ judge_model,
+ error = %error,
+ "judge verdict unavailable; routing without one"
+ );
+ };
+
+ let response = driver
+ .call_llm_target(
+ Context::default(),
+ &self.target,
+ self.judge.build_request(state, request),
+ Arc::new(JudgeDecision {
+ model: self.target.semantic_name.to_string(),
+ }),
+ )
+ .await
+ .inspect_err(|error| warn(error))
+ .ok()?;
+ let aggregate = response
+ .llm_response
+ .into_agg()
+ .await
+ .inspect_err(|error| warn(error))
+ .ok()?;
+ self.judge
+ .parse(&aggregate)
+ .inspect_err(|error| warn(error))
+ .ok()
+ }
+}
+
+#[async_trait]
+impl<J, P> Classifier for JudgeClassifier<J, P>
+where
+ J: Judge,
+ P: JudgePolicy<Verdict = J::Verdict>,
+{
+ async fn score(
+ &self,
+ state: &mut State,
+ request: &Request,
+ driver: Option<&Driver>,
+ ) -> Result<Classification> {
+ // A missing driver is a broken composition, not an unavailable judge.
+ let Some(driver) = driver else {
+ return Err(LibsyError::AlgorithmError {
+ message: format!(
+ "judge classifier for target {:?} requires a driver to call it",
+ self.target.semantic_name
+ ),
+ });
+ };
+ let verdict = self.verdict(state, request, driver).await;
+ Ok(self.policy.to_classification(verdict.as_ref()))
+ }
+}
+
+fn parse_json_verdict<T: DeserializeOwned>(response: &AggLlmResponse) -> Result<T> {
+ // Providers sometimes wrap otherwise valid JSON in a Markdown fence.
+ let completion = completion_text(response);
+ serde_json::from_str(strip_json_fence(completion.trim())).map_err(|err| {
+ LibsyError::AlgorithmError {
+ message: format!(
+ "judge reply did not parse as {}: {err}",
+ std::any::type_name::<T>()
+ ),
+ }
+ })
+}
+
+struct JudgeDecision {
+ model: String,
+}
+
+impl Decision for JudgeDecision {
+ fn selected_model(&self) -> &str {
+ &self.model
+ }
+
+ fn reasoning(&self) -> Option<&str> {
+ Some("llm judge consultation")
+ }
+
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+}
+
+fn strip_json_fence(text: &str) -> &str {
+ let Some(rest) = text.strip_prefix("```") else {
+ return text;
+ };
+ let rest = rest.strip_prefix("json").unwrap_or(rest);
+ let rest = rest.trim_start_matches(['\n', '\r']);
+ rest.strip_suffix("```").map(str::trim).unwrap_or(rest)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use futures::StreamExt;
+ use serde::Deserialize;
+ use switchyard_protocol::{text_request, text_response, LlmClientError};
+
+ use crate::{LlmResponse, LlmResponseChunk, Response, Score, Step};
+
+ const VERDICT: &str = r#"{"ok":true}"#;
+
+ #[derive(Debug, Deserialize, PartialEq)]
+ struct TestVerdict {
+ ok: bool,
+ }
+
+ struct TestJudge;
+
+ impl Judge for TestJudge {
+ type Verdict = TestVerdict;
+
+ fn build_request(&self, _state: &State, request: &Request) -> Request {
+ request.clone()
+ }
+ }
+
+ /// Reports only whether a verdict arrived.
+ struct TestPolicy;
+
+ impl JudgePolicy for TestPolicy {
+ type Verdict = TestVerdict;
+
+ fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification {
+ let target = if verdict.is_some() {
+ "verdict"
+ } else {
+ "no-verdict"
+ };
+ Classification::Scores(vec![Score {
+ target: target.to_string(),
+ confidence: 1.0,
+ }])
+ }
+ }
+
+ fn classifier() -> JudgeClassifier<TestJudge, TestPolicy> {
+ JudgeClassifier::new(
+ TestJudge,
+ LlmTarget {
+ semantic_name: "judge".to_string(),
+ llm_client: None,
+ },
+ TestPolicy,
+ )
+ }
+
+ fn request() -> Request {
+ Request {
+ llm_request: text_request(Some("auto".to_string()), "judge this"),
+ raw_request: None,
+ metadata: None,
+ }
+ }
+
+ fn buffered(completion: &str) -> Response {
+ Response {
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
+ metadata: None,
+ }
+ }
+
+ fn streamed(chunks: Vec<LlmResponseChunk>) -> Response {
+ Response {
+ llm_response: LlmResponse::Stream(
+ futures::stream::iter(chunks.into_iter().map(Ok)).boxed(),
+ ),
+ metadata: None,
+ }
+ }
+
+ fn streamed_then_failing(chunk: LlmResponseChunk) -> Response {
+ let items = futures::stream::iter([
+ Ok(chunk),
+ Err(LlmClientError::Timeout {
+ source: Box::new(std::io::Error::other("stream died")),
+ }),
+ ]);
+ Response {
+ llm_response: LlmResponse::Stream(items.boxed()),
+ metadata: None,
+ }
+ }
+
+ fn selected(classification: Classification) -> Result<String> {
+ classification
+ .argmax(false)?
+ .map(|score| score.target)
+ .ok_or_else(|| LibsyError::AlgorithmError {
+ message: "policy abstained".to_string(),
+ })
+ }
+
+ /// Serves the single offloaded judge call with `reply`. The stream is taken first
+ /// because the driver refuses to publish a step until a consumer exists.
+ async fn score_served_with(reply: Result<Response>) -> Result<String> {
+ let driver = Driver::new();
+ let mut steps = Box::pin(driver.stream());
+ let classifier = classifier();
+ let mut state = State::default();
+ let request = request();
+
+ let serve = async {
+ if let Some(Ok(Step::CallLlm(call))) = steps.next().await {
+ let _ = call.respond(reply);
+ }
+ };
+ let (classification, ()) =
+ tokio::join!(classifier.score(&mut state, &request, Some(&driver)), serve);
+ selected(classification?)
+ }
+
+ #[tokio::test]
+ async fn a_buffered_verdict_reaches_the_policy() -> Result<()> {
+ assert_eq!(score_served_with(Ok(buffered(VERDICT))).await?, "verdict");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn a_streamed_verdict_is_drained_before_parsing() -> Result<()> {
+ let chunks = VERDICT
+ .chars()
+ .map(|character| LlmResponseChunk::TextDelta {
+ index: 0,
+ text: character.to_string(),
+ })
+ .collect();
+ assert_eq!(score_served_with(Ok(streamed(chunks))).await?, "verdict");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn an_in_band_stream_error_falls_back_to_the_policy() -> Result<()> {
+ let chunks = vec![
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "{\"ok\":".to_string(),
+ },
+ LlmResponseChunk::StreamError {
+ message: "upstream exploded".to_string(),
+ },
+ ];
+ assert_eq!(score_served_with(Ok(streamed(chunks))).await?, "no-verdict");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn a_transport_failure_mid_stream_falls_back_to_the_policy() -> Result<()> {
+ let partial = LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "{\"ok\":".to_string(),
+ };
+ assert_eq!(
+ score_served_with(Ok(streamed_then_failing(partial))).await?,
+ "no-verdict"
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn an_unparseable_reply_falls_back_to_the_policy() -> Result<()> {
+ assert_eq!(
+ score_served_with(Ok(buffered("sorry, I can't help with that"))).await?,
+ "no-verdict"
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn a_failed_judge_call_falls_back_to_the_policy() -> Result<()> {
+ let error = LibsyError::client_call(
+ "judge",
+ LlmClientError::Timeout {
+ source: Box::new(std::io::Error::other("judge unreachable")),
+ },
+ );
+ assert_eq!(score_served_with(Err(error)).await?, "no-verdict");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn a_missing_driver_is_an_error_not_a_fallback() -> Result<()> {
+ let error = classifier()
+ .score(&mut State::default(), &request(), None)
+ .await
+ .err()
+ .ok_or_else(|| LibsyError::AlgorithmError {
+ message: "expected a missing-driver error".to_string(),
+ })?;
+
+ assert!(
+ matches!(&error, LibsyError::AlgorithmError { message } if message.contains("judge")),
+ "unexpected error: {error}"
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn fenced_replies_parse_as_verdicts() -> Result<()> {
+ let judge = TestJudge;
+ for reply in ["```json\n{\"ok\":true}\n```", "```\n{\"ok\":true}\n```"] {
+ assert!(judge.parse(&text_response(None, reply))?.ok);
+ }
+ Ok(())
+ }
+}
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index c59a48e0c..68bddd10a 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -74,8 +74,8 @@
//!
//! [`algorithms::Random`] provides uniform or weighted random routing.
//!
-//! [`algorithms::LlmClassifier`] classifies with one model, then routes to a strong/weak
-//! model depending on the classifier's choice.
+//! [`algorithms::LlmTaskClassifier`] classifies with one model; compose it with
+//! [`algorithms::FallThrough`] to route to the selected target.
mod core;
pub use core::*;
diff --git a/crates/libsy/src/prompts/.gitkeep b/crates/libsy/src/prompts/.gitkeep
new file mode 100644
index 000000000..e69de29bb
diff --git a/crates/libsy/src/prompts/capability-classifier/prompt.md b/crates/libsy/src/prompts/capability-classifier/prompt.md
new file mode 100644
index 000000000..fdc983e30
--- /dev/null
+++ b/crates/libsy/src/prompts/capability-classifier/prompt.md
@@ -0,0 +1,74 @@
+You are a task-level routing classifier for an agent harness. You receive only
+the task's initial instruction. Estimate whether the efficient agent, GLM-5.2,
+will complete the whole task correctly. Routing is sticky: the selected agent
+owns the task through completion.
+
+Use only conditions visible in the instruction. Never assume hidden environment
+facts, artifact integrity, tools, dependencies, tests, or access. Capability is
+determined by verification, boundedness, and closure risk rather than topic or
+component count.
+
+# Definitions
+
+- Authoritative verification is independent_full when an evaluator, reference,
+ checksum, regression suite, replay, or end-to-end client independent of the
+ proposed solution checks the complete final output or state.
+- Verification is partial_or_self_authored when it is a smoke test, a check the
+ solver must invent, or covers only part of correctness.
+- Verification is hidden_or_private when final acceptance uses inaccessible
+ data, edge cases, thresholds, rankings, or conventions. A stated public check
+ or numeric target does not make private acceptance independent_full.
+- Verification is not_stated when the instruction names no way to falsify a
+ wrong final result. Do not infer that a repository contains tests.
+- An inverse, recovery, or search is bounded only when the instruction exposes
+ a finite or tightly constrained hypothesis space, source or format structure,
+ rich reference pairs, a checksum, or an authoritative replay check. A topic
+ such as cryptanalysis or reverse engineering is not inherently unbounded.
+- Closure risk is the risk that substantial exploration or a chain of fixes
+ will not be converted into the saved, re-run, fully verified final artifact.
+ Several components with an early end-to-end acceptance endpoint can be
+ bounded; an open-ended compatibility campaign is not bounded merely because
+ a final test command exists.
+
+# Assessment procedure
+
+1. State the hardest requirement without assuming hidden environment facts.
+2. Classify the authoritative verification visible in the instruction using
+ the definitions above.
+3. Decide whether every inverse, recovery, or search is bounded and
+ independently testable.
+4. Check whether the deliverable must work in a clean environment under stated
+ dependency constraints.
+5. Estimate exploration and submission-closure risk separately from conceptual
+ difficulty or the number of named components.
+6. Select the one capability rule that best matches those conditions. Set
+ capability_boundary to that rule's assigned boundary. Rule ids are opaque;
+ never infer meaning from their spelling. Use unmatched with primary_rule=none
+ when no rule applies or instruction-visible evidence is insufficient.
+7. Estimate p_solve for whole-task completion, then make a quality-first route
+ recommendation. Recommend efficient only when the evidence strongly covers
+ the crux. Recommend capable for unstable, hidden-acceptance, latent-state, or
+ material closure-risk cases.
+
+Set abstain=true only when the instruction is too incomplete to assess. Set
+confidence to confidence in this assessment, not probability of task success.
+Do not encode cost preferences in p_solve or recommended_route.
+
+# Capability boundaries
+
+[CAPABILITY_CARD]
+- SUP-1 [supported]: Deterministic extraction, validation, counting, aggregation, normalization, exact-schema generation, or single-regex construction when input formats, boundary rules, tie-breaking, reference conventions, and output ordering are explicit and locally checkable; exclude learned-model similarity/ranking, evolving external sources, or multi-hop ontology queries whose semantic joins and status rules must be inferred.
+- SUP-2 [supported]: Conventional local software, service, dependency, build, proof, or artifact implementation/repair, including simple RPC services and compiling specified or publicly locatable source, when the required interface, files, commands, ports, invariants, or sanity checks are explicit and a local build, compiler, unit check, render command, or exact output check can directly validate completion.
+- SUP-3 [supported]: Bounded reverse engineering, cryptanalysis, or artifact recovery when the instruction supplies source, a chosen oracle, or a standard self-describing format; the requested state is small or structurally constrained, and parser validation, replay, checksum, coverage scoring, or local oracle checks can validate the result. Includes standard executable segment/value extraction and toy-cipher key-component recovery; exclude unknown storage remnants unless the search is explicitly small and verified.
+- UNC-1 [uncertain]: Exact-answer tasks involving external or historical leaderboards, learned embedding/model rankings, computer-vision event timing, statistical, causal, accuracy, or performance claims, or schema-heavy semantic queries where correctness depends on hidden/private data, evolving web state, sampling variability, model/library conventions, inferred ontology semantics, or unspecified tolerances; explicit formal specifications with direct local edge-case tests are excluded.
+- UNC-2 [uncertain]: Byte-identical reimplementation or porting of a program with persistent file mutations where correctness depends on legacy or runtime-specific record I/O, padding, numeric encodings, short-record handling, or update ordering, even if source and an executable comparison are available.
+- UNC-3 [uncertain]: Independent source reconstruction or broad behavior recovery for an unknown compiled program or black-box artifact with no exhaustive specification; observed I/O or disassembly gives only partial confidence unless the task is tightly bounded and exactly verified. Do not apply to standard-format metadata/segment extraction or provided-source bounded cryptanalysis.
+- UNC-4 [uncertain]: Deleted-file or forensic password recovery from filesystem remnants, disk images, fragmented archives, or unknown storage layers when the instruction gives only filename or pattern constraints and recovery may require carving, fragment assembly, checksum reasoning, or large search over missing bytes.
+- LIM-1 [unsupported]: Exact reconstruction or semantic equivalence over unbounded hypotheses with no source, bounded search space, rich reference pairs, checksum, replay, or exhaustive oracle; do not apply when the instruction supplies enough structure to make the inverse testable.
+[/CAPABILITY_CARD]
+
+# Output
+
+Return exactly one JSON object with no markdown or commentary:
+
+{{RESPONSE_SCHEMA}}
diff --git a/crates/libsy/src/prompts/capability-classifier/schema.json b/crates/libsy/src/prompts/capability-classifier/schema.json
new file mode 100644
index 000000000..6c4e2f02d
--- /dev/null
+++ b/crates/libsy/src/prompts/capability-classifier/schema.json
@@ -0,0 +1,21 @@
+{
+ "type": "json_schema",
+ "json_schema": {
+ "name": "CapabilityClassifierDecision",
+ "strict": true,
+ "schema": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["recommended_route", "p_solve", "confidence", "abstain", "capability_boundary", "primary_rule", "crux"],
+ "properties": {
+ "recommended_route": {"type": "string", "enum": ["efficient", "capable"]},
+ "p_solve": {"type": "number", "minimum": 0.0, "maximum": 1.0},
+ "confidence": {"type": "number", "minimum": 0.0, "maximum": 1.0},
+ "abstain": {"type": "boolean"},
+ "capability_boundary": {"type": "string", "enum": ["supported", "uncertain", "unsupported", "unmatched"]},
+ "primary_rule": {"type": "string", "enum": ["SUP-1", "SUP-2", "SUP-3", "UNC-1", "UNC-2", "UNC-3", "UNC-4", "LIM-1", "none"]},
+ "crux": {"type": "string", "minLength": 1}
+ }
+ }
+ }
+}
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index f1044b346..753333d32 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -13,6 +13,7 @@ rust-version.workspace = true
[dependencies]
async-stream.workspace = true
+async-trait.workspace = true
axum = "0.8"
axum-server = { version = "0.8", default-features = false, features = ["tls-rustls"] }
rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs", "std", "tls12"] }
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index f7a0eba6d..0c6ac9559 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -8,8 +8,8 @@ use std::fs;
use std::path::Path;
use std::sync::Arc;
-use libsy::algorithms::{LlmClassifier, Noop, Passthrough, Random};
-use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
+use libsy::algorithms::{FallThrough, LlmTaskClassifier, Noop, Passthrough, Random};
+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, SharedState};
use serde::Deserialize;
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
@@ -240,7 +240,7 @@ fn build_algorithm(
route_name: &str,
config: &RouteConfig,
targets: &BTreeMap<String, LlmTarget>,
-) -> ServerResult<Arc<dyn Algorithm>> {
+) -> ServerResult<Arc<dyn Algorithm<SharedState>>> {
match config {
RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})),
RouteConfig::Random {
@@ -269,19 +269,15 @@ fn build_algorithm(
let classifier = resolve_target(route_name, classifier_target, targets)?;
let strong = resolve_target(route_name, strong_target, targets)?;
let weak = resolve_target(route_name, weak_target, targets)?;
- let target_set =
- LlmTargetSet::new(vec![classifier.clone(), strong.clone(), weak.clone()]);
- let algorithm = LlmClassifier::new(
- classifier.semantic_name,
- strong.semantic_name,
- weak.semantic_name,
- *threshold,
- target_set,
- )
- .map_err(|error| {
- ServerError::new(format!("llm_classifier route {route_name}: {error}"))
- })?;
- Ok(Arc::new(algorithm))
+ // The judge is called through its own target, so it is not a routing destination.
+ let target_set = LlmTargetSet::new(vec![strong.clone(), weak.clone()]);
+ // The weak model is the efficient tier; the strong model is the capable one.
+ let judge =
+ LlmTaskClassifier::new(classifier, weak, strong, *threshold).map_err(|error| {
+ ServerError::new(format!("llm_classifier route {route_name}: {error}"))
+ })?;
+ let router = FallThrough::new(target_set).with_classifier(Arc::new(judge));
+ Ok(Arc::new(router))
}
}
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index dbcb663c5..5221b3358 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -23,7 +23,9 @@ use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Json, Router};
use axum_server::tls_rustls::RustlsConfig;
-use libsy::{Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request};
+use libsy::{
+ Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request, SharedState,
+};
use serde_json::{json, Value};
use tokio::net::{TcpListener, TcpSocket};
use tracing::Level;
@@ -68,14 +70,16 @@ pub type ServerResult<T> = std::result::Result<T, ServerError>;
/// Shared server state used by all endpoint handlers.
#[derive(Clone)]
pub struct ServerState {
- routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,
+ /// For now making all the routes to be of type Algorithm<SharedState>
+ /// Nachi's PR will help later
+ routes: Arc<BTreeMap<String, Arc<dyn Algorithm<SharedState>>>>,
metrics: prometheus::Registry,
}
impl ServerState {
/// Creates server state from route model IDs and their libsy algorithms.
pub fn new(
- routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>)>,
+ routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm<SharedState>>)>,
) -> ServerResult<Self> {
let mut entries = BTreeMap::new();
for (model, algorithm) in routes {
@@ -102,7 +106,7 @@ impl ServerState {
self.routes.keys().map(String::as_str)
}
- fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm>> {
+ fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm<SharedState>>> {
self.routes.get(model).map(Arc::clone)
}
}
@@ -333,7 +337,10 @@ async fn handle_llm_request(
raw_request: Some(body),
metadata: Some(metadata),
};
- let (trace, response) = match algorithm.run(Context::default(), request).await {
+ let (trace, response) = match algorithm
+ .run(Context::<SharedState>::default(), request)
+ .await
+ {
Ok(result) => result,
Err(error) => return algorithm_error(error),
};
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index b7a6abbc1..54aad67f6 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -18,7 +18,7 @@ use axum::routing::post;
use axum::{Json, Router};
use http_body_util::BodyExt;
use libsy::algorithms::Random;
-use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, SharedState};
use serde_json::{json, Value};
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
use switchyard_server::config::load_server_state;
@@ -96,7 +96,7 @@ async fn upstream_chat(
}
let content = if model == "model/classifier" {
- "0.9"
+ r#"{"recommended_route":"efficient","p_solve":0.9,"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#
} else {
"ok"
};
@@ -146,7 +146,8 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server
})
.collect(),
);
- let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set, None, None)?);
+ let algorithm: Arc<dyn Algorithm<SharedState>> =
+ Arc::new(Random::new(target_set, None, None)?);
Ok(((*route_model).to_string(), algorithm))
})
.collect::<TestResult<Vec<_>>>()?;
@@ -295,7 +296,7 @@ target = "weak"
for (route, selected) in [
("switchyard/random", "model/weak"),
- ("switchyard/classifier", "model/strong"),
+ ("switchyard/classifier", "model/weak"),
("switchyard/passthrough", "model/weak"),
] {
let response = send(
@@ -322,7 +323,7 @@ target = "weak"
assert_eq!(calls.len(), 4);
assert_eq!(calls[0]["model"], "model/weak");
assert_eq!(calls[1]["model"], "model/classifier");
- assert_eq!(calls[2]["model"], "model/strong");
+ assert_eq!(calls[2]["model"], "model/weak");
assert_eq!(calls[3]["model"], "model/weak");
Ok(())
}