Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): Externalize State so that it survives restarts (#1024)

Graham Kingd93bcd974a7f
diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml
index 08a885535..027242ec2 100644
--- a/.github/workflows/commitlint.yml
+++ b/.github/workflows/commitlint.yml
@@ -19,13 +19,6 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- - uses: actions/setup-node@v4
+ - uses: wagoid/commitlint-github-action@v6
with:
- node-version: "22.23.3"
- - name: Install Commitlint
- run: npm install --no-save --package-lock=false --ignore-scripts @commitlint/[email protected] @commitlint/[email protected]
- - name: Commit messages
- env:
- COMMITLINT_FROM: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
- COMMITLINT_TO: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }}
- run: npx --no-install commitlint --config .commitlintrc.json --from "$COMMITLINT_FROM" --to "$COMMITLINT_TO" --verbose
+ configFile: .commitlintrc.json
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2d43f0d47..b79ec9d58 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,7 +11,15 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
- **VGR routing (planned for 0.4.0)** — adds `type = "vgr"` deployment routes
and the `AlgorithmSpec::Vgr` variant. **Rust source-breaking change:** callers
that match exhaustively on `AlgorithmSpec` must handle the new variant.
-
+- **Caller-owned routing state** - `Algorithm::run_stream` takes a `State` and
+ `RoutingOutcome.state` returns the updated `State`, with `Serialize` and
+ `Deserialize` on `State` and `StateValue` for hosts that store routing state
+ across turns outside the router.
+ **Host API change (source-breaking):** `Processor`, `Classifier`,
+ `FallThrough` and `SubagentGate` use `State` in place of a type parameter.
+ `State.extra` is a `BTreeMap`. `State.turn_count` and `DECISION_SOURCE_KEY`
+ are removed. `StateValue::Bool` replaces `StateValue::Int` and
+ `StateValue::Scalar`.
- **Speculative workflows** - algorithms can run model and decision calls in
parallel, stop one call, and let another continue. This includes decision
models used as speculative tool callers. The same controls support general
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 24f32935b..27a5f3862 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -742,6 +742,7 @@ async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyard
let mut steps = router.clone().run_stream(
request.clone(),
classifier_models("warning/judge", "warning/efficient", "warning/capable"),
+ switchyard_libsy::State::default(),
);
let _ = steps.next().await;
}
@@ -1704,6 +1705,7 @@ async fn decision_calls_record_each_terminal_path_once() -> switchyard_libsy::Re
let mut stream = Box::pin(Arc::new(DecisionAlgo(algorithm.clone())).run_stream(
request_with_metadata("decision-session", "decision-correlation"),
Arc::new(RuntimeModels::default()),
+ switchyard_libsy::State::default(),
));
let Some(Ok(Step::CallDecision(call))) = stream.next().await else {
return Err(test_error("expected a decision call"));
@@ -1819,6 +1821,7 @@ async fn failed_call_records_metrics_without_error_details() -> switchyard_libsy
let stream = algorithm.run_stream(
request_with_metadata("obs-session-2", "obs-corr-2"),
Arc::new(RuntimeModels::default()),
+ switchyard_libsy::State::default(),
);
tokio::pin!(stream);
@@ -2186,6 +2189,7 @@ async fn in_flight_gauge_reads_a_run_parked_on_an_unanswered_routing_call()
let stream = algorithm.run_stream(
request_with_metadata("obs-session-if", "obs-corr-if"),
Arc::new(RuntimeModels::default()),
+ switchyard_libsy::State::default(),
);
tokio::pin!(stream);
let attributes = [("algorithm", ALGO)];
@@ -2246,6 +2250,7 @@ async fn in_flight_gauge_clears_when_a_run_is_abandoned() -> switchyard_libsy::R
let stream = algorithm.run_stream(
request_with_metadata("obs-session-ab", "obs-corr-ab"),
Arc::new(RuntimeModels::default()),
+ switchyard_libsy::State::default(),
);
tokio::pin!(stream);
let Some(Ok(Step::CallModel(_call))) = stream.next().await else {
diff --git a/crates/libsy/src/algorithms/advisor_gate.rs b/crates/libsy/src/algorithms/advisor_gate.rs
index fde8dc6ad..d44711e8d 100644
--- a/crates/libsy/src/algorithms/advisor_gate.rs
+++ b/crates/libsy/src/algorithms/advisor_gate.rs
@@ -39,7 +39,7 @@ use switchyard_protocol::{
};
use crate::core::algorithm::{Algorithm, Driver, RoutingOutcome};
-use crate::core::processor::{Event, Processor};
+use crate::core::processor::Event;
use crate::{LibsyError, Result};
mod budget;
diff --git a/crates/libsy/src/algorithms/advisor_gate/signals.rs b/crates/libsy/src/algorithms/advisor_gate/signals.rs
index 9cb36f477..8209a8125 100644
--- a/crates/libsy/src/algorithms/advisor_gate/signals.rs
+++ b/crates/libsy/src/algorithms/advisor_gate/signals.rs
@@ -5,11 +5,9 @@
//! fold in on [`Event::Request`], the generated turn's shape on
//! [`Event::ModelResponse`], and the trigger classifier reads both.
-use async_trait::async_trait;
-
use crate::Result;
use crate::algorithms::util::tool_signals::ToolSignals;
-use crate::core::processor::{Event, Processor};
+use crate::core::processor::Event;
use super::turn::{has_tool_use, visible_text};
@@ -34,9 +32,8 @@ pub(super) struct TurnSignals {
/// Fills [`GateSignals`], each event writing its own side.
pub(super) struct GateSignalProcessor;
-#[async_trait]
-impl Processor<GateSignals> for GateSignalProcessor {
- async fn process(&self, state: &mut GateSignals, event: Event<'_>) -> Result<()> {
+impl GateSignalProcessor {
+ pub(super) async fn process(&self, state: &mut GateSignals, event: Event<'_>) -> Result<()> {
match event {
Event::Request { request, .. } => {
state.conversation = ToolSignals::from_request(request, None);
diff --git a/crates/libsy/src/algorithms/composite.rs b/crates/libsy/src/algorithms/composite.rs
index a054d7ac5..1ce565dff 100644
--- a/crates/libsy/src/algorithms/composite.rs
+++ b/crates/libsy/src/algorithms/composite.rs
@@ -18,7 +18,7 @@ use super::llm_class::{
};
use super::stage::{StageRouterConfig, build_stage_route};
use super::util::affinity::{ClassifyTrigger, evict_if_full, has_new_user_turn, retention_key};
-use super::util::stage::{Tier, set_fall_open};
+use super::util::stage::{Tier, fall_open_tier, set_fall_open};
use crate::core::algorithm::{Algorithm, Driver, RoutingIdentity};
use crate::core::classifier::Classifier;
use crate::core::processor::{Event, Processor};
@@ -34,7 +34,7 @@ const COMPOSITE: &str = "composite";
/// session ID when `message_hash_fallback` is on. The retained tier is replayed
/// into state on every request so the cascade below reads it.
struct TierSetter {
- judge: Arc<dyn Classifier<State>>,
+ judge: Arc<dyn Classifier>,
trigger: ClassifyTrigger,
message_hash_fallback: bool,
default_tier: Tier,
@@ -52,12 +52,14 @@ impl TierSetter {
/// Two requests for one identity can both pass this and both judge, since a
/// judge call sits between here and [`retain`](Self::retain). The later wins.
- fn is_due(&self, identity: Option<&RoutingIdentity>, request: &Request) -> bool {
+ fn is_due(&self, identity: Option<&RoutingIdentity>, state: &State, request: &Request) -> bool {
match self.trigger {
ClassifyTrigger::UserTurn => has_new_user_turn(&request.llm_request.messages),
- // Unkeyed requests cannot be told apart, so every one is a new session.
+ // Unkeyed requests with no carried tier cannot be told apart, so every one
+ // is a new session.
ClassifyTrigger::NewSession => {
identity.is_none_or(|identity| !self.tiers.lock().contains_key(identity))
+ && fall_open_tier(state).is_none()
}
// Rejected by the constructor, and only in the enum for the standalone route.
ClassifyTrigger::EveryRequest => true,
@@ -77,13 +79,13 @@ impl TierSetter {
}
#[async_trait]
-impl Processor<State> for TierSetter {
+impl Processor for TierSetter {
async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
let Event::Request { request, driver } = event else {
return Ok(());
};
let identity = retention_key(request, self.message_hash_fallback);
- if self.is_due(identity.as_ref(), request) {
+ if self.is_due(identity.as_ref(), state, request) {
let (classification, _) = self.judge.score(state, request, driver).await?;
if let Some(winner) = classification.argmax(false)?
&& let Some(tier) = Self::tier_for(winner.category)
@@ -103,7 +105,9 @@ impl Processor<State> for TierSetter {
}
// Either not this request's turn or the judge had no verdict, so the last
// tier stands rather than dropping back to the picker default.
- if let Some(tier) = identity.and_then(|identity| self.tiers.lock().get(&identity).copied())
+ if let Some(tier) = identity
+ .and_then(|identity| self.tiers.lock().get(&identity).copied())
+ .or_else(|| fall_open_tier(state))
{
set_fall_open(state, tier);
driver.set_evidence_if_empty(serde_json::json!({"source": "retained"}));
@@ -130,7 +134,7 @@ pub struct CompositeRouterConfig {
/// Runs a stage router with a tier the judge picks.
pub struct CompositeRouter {
- route: FallThrough<State>,
+ route: FallThrough,
}
impl CompositeRouter {
@@ -154,7 +158,7 @@ impl CompositeRouter {
message: "composite: classify_trigger must be user_turn or new_session".to_string(),
});
}
- let judge: Arc<dyn Classifier<State>> = match config.judge {
+ let judge: Arc<dyn Classifier> = match config.judge {
CompositeJudgeConfig::Capability(judge) => {
// Composite owns the trigger and retention; standalone affinity never runs.
Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
@@ -257,13 +261,17 @@ mod tests {
}
fn router() -> Result<Arc<CompositeRouter>> {
+ router_with_trigger(ClassifyTrigger::UserTurn)
+ }
+
+ fn router_with_trigger(classify_trigger: ClassifyTrigger) -> Result<Arc<CompositeRouter>> {
Ok(Arc::new(CompositeRouter::new(CompositeRouterConfig {
judge: CompositeJudgeConfig::Capability(TaskClassifierConfig {
judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
base_threshold: 0.5,
..LlmCapabilityConfig::default()
}),
- classify_trigger: ClassifyTrigger::UserTurn,
+ classify_trigger,
..Default::default()
}),
stage: StageRouterConfig::new(PickerMode::EfficientFirst, 0.5),
@@ -530,4 +538,34 @@ mod tests {
);
Ok(())
}
+
+ #[tokio::test]
+ async fn caller_state_carries_the_judged_tier_without_a_session_id() -> Result<()> {
+ for trigger in [ClassifyTrigger::UserTurn, ClassifyTrigger::NewSession] {
+ let recorder = Arc::new(Recorder::default());
+ *recorder.judge_p_solve.lock() = 0.1;
+ let router = router_with_trigger(trigger)?;
+ let run = |request, state| {
+ crate::core::testing::test_drive_with_state(
+ router.clone(),
+ request,
+ runtime_models(),
+ state,
+ recorder.serve(),
+ )
+ };
+
+ let first = run(unkeyed(user_turn_request()), State::default()).await?;
+ assert_eq!(first.selected_model_id()?, &ModelId::from("strong"));
+
+ let continued = run(unkeyed(turn_request(false)), first.state).await?;
+ assert_eq!(continued.selected_model_id()?, &ModelId::from("strong"));
+ assert_eq!(
+ continued.metadata.and_then(|m| m.evidence),
+ Some(serde_json::json!({"source": "retained"}))
+ );
+ assert_eq!(recorder.judge_calls(), 1, "{trigger:?}");
+ }
+ Ok(())
+ }
}
diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs
index cced4d9fb..41dce8677 100644
--- a/crates/libsy/src/algorithms/escalation.rs
+++ b/crates/libsy/src/algorithms/escalation.rs
@@ -89,7 +89,7 @@ pub(super) fn build_classifier(
contract_config: ClassifierContractConfig,
config: EscalationJudgeConfig,
max_output_tokens: u64,
-) -> Result<Arc<dyn Classifier<State>>> {
+) -> Result<Arc<dyn Classifier>> {
let confirmations = config.confirmations;
let deescalation = match config.deescalation {
Some(deescalation) => Some(DeescalationPolicy {
@@ -104,7 +104,7 @@ pub(super) fn build_classifier(
None => None,
};
let is_phase_aware = deescalation.is_some();
- let classifier: Arc<dyn Classifier<State>> = Arc::new(EscalationClassifier {
+ let classifier: Arc<dyn Classifier> = Arc::new(EscalationClassifier {
escalation_judge: escalation::build_judge(
&contract_config,
config,
@@ -204,7 +204,7 @@ impl EscalationClassifier {
}
#[async_trait]
-impl Classifier<State> for EscalationClassifier {
+impl Classifier for EscalationClassifier {
async fn score(
&self,
state: &mut State,
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 8d6f5e71d..eb8198a91 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -9,9 +9,9 @@
//! (its `argmax`); the selected model is replayed to the processors so
//! stateful ones (latch, affinity) can bind it.
//!
-//! The default `FallThrough<()>` carries no composition state. Stateful compositions share one
-//! private state value across turns with the same session ID. Requests without a session ID use
-//! unretained per-run state.
+//! The default `FallThrough` carries no session registry. Stateful compositions share one
+//! private [`State`] across turns with the same session ID. Requests without a session ID run
+//! against the caller state on the [`Driver`].
//!
//! The selected target is offered first, then the rest of the category it was drawn from, then
//! every other runtime target. The consumer may fall through that ordered candidate list when a
@@ -30,15 +30,16 @@ use tokio::sync::Mutex as AsyncMutex;
use crate::core::algorithm::{self, Algorithm, Driver};
use crate::core::classifier::{Classifier, Score};
use crate::core::processor::{Event, Processor};
+use crate::core::state::State;
use crate::{LibsyError, Result, RoutingOutcome};
use switchyard_protocol::{Category, ModelId, Request, Response};
-struct SessionState<S> {
- state: Arc<AsyncMutex<S>>,
+struct SessionState {
+ state: Arc<AsyncMutex<State>>,
last_accessed: Instant,
}
-type SessionStates<S> = Mutex<HashMap<String, SessionState<S>>>;
+type SessionStates = Mutex<HashMap<String, SessionState>>;
/// Delete sessions that have been inactive this long. Catches sessions that did not terminate
/// cleanly.
@@ -50,17 +51,15 @@ const SESSION_STATE_TTL: Duration = Duration::from_secs(60 * 60);
const SESSION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 60);
/// Processor chain → classifier cascade → routed model call. See the module docs.
-///
-/// The generic state type is shared by every processor and classifier in the composition.
-pub struct FallThrough<S = ()> {
+pub struct FallThrough {
name: String,
- processors: Vec<Arc<dyn Processor<S>>>,
- classifiers: Vec<Arc<dyn Classifier<S>>>,
- session_states: Option<Arc<SessionStates<S>>>,
+ processors: Vec<Arc<dyn Processor>>,
+ classifiers: Vec<Arc<dyn Classifier>>,
+ session_states: Option<Arc<SessionStates>>,
cleanup_started: Once,
}
-impl FallThrough<()> {
+impl FallThrough {
/// Creates an empty stateless router.
pub fn new() -> Self {
Self {
@@ -71,13 +70,8 @@ impl FallThrough<()> {
cleanup_started: Once::new(),
}
}
-}
-impl<S> FallThrough<S>
-where
- S: Default + Send + 'static,
-{
- /// Creates a router that retains one private `S` per session.
+ /// Creates a router that retains one private [`State`] per session.
pub fn new_with_state() -> Self {
Self {
name: "fall_through".to_string(),
@@ -95,20 +89,22 @@ where
}
/// Appends a processor to the head-of-request chain.
- pub fn with_processor(mut self, processor: Arc<dyn Processor<S>>) -> Self {
+ pub fn with_processor(mut self, processor: Arc<dyn Processor>) -> Self {
self.processors.push(processor);
self
}
/// Appends a classifier to the cascade.
- pub fn with_classifier(mut self, classifier: Arc<dyn Classifier<S>>) -> Self {
+ pub fn with_classifier(mut self, classifier: Arc<dyn Classifier>) -> Self {
self.classifiers.push(classifier);
self
}
/// Executes the processor and classifier sequence for wrappers and the trait entrypoint.
pub(crate) async fn execute(&self, driver: Driver, request: Request) -> Result<RoutingOutcome> {
- self.start_cleanup_task();
let session = session_id(&request);
+ if session.is_some() {
+ self.start_cleanup_task();
+ }
let session_final = request
.metadata
.as_ref()
@@ -143,8 +139,10 @@ where
self.route(&mut state, &driver, &mut request).await?
}
None => {
- let mut state = S::default();
- self.route(&mut state, &driver, &mut request).await?
+ let mut state = driver.take_state();
+ let routed = self.route(&mut state, &driver, &mut request).await?;
+ driver.set_state(state);
+ routed
}
};
@@ -181,13 +179,13 @@ where
}
/// Returns this request's retained state without holding the registry lock.
- fn session_state(&self, request: &Request) -> Option<Arc<AsyncMutex<S>>> {
+ fn session_state(&self, request: &Request) -> Option<Arc<AsyncMutex<State>>> {
let states = self.session_states.as_ref()?;
let session_id = session_id(request)?;
let mut states = states.lock();
let now = Instant::now();
let session = states.entry(session_id).or_insert_with(|| SessionState {
- state: Arc::new(AsyncMutex::new(S::default())),
+ state: Arc::new(AsyncMutex::new(State::default())),
last_accessed: now,
});
session.last_accessed = now;
@@ -196,7 +194,7 @@ where
async fn route(
&self,
- state: &mut S,
+ state: &mut State,
driver: &Driver,
request: &mut Request,
) -> Result<(Score, Option<Response>)> {
@@ -246,10 +244,7 @@ where
}
}
-async fn cleanup_inactive_sessions<S>(states: Weak<SessionStates<S>>)
-where
- S: Send + 'static,
-{
+async fn cleanup_inactive_sessions(states: Weak<SessionStates>) {
let start = tokio::time::Instant::now() + SESSION_CLEANUP_INTERVAL;
let mut interval = tokio::time::interval_at(start, SESSION_CLEANUP_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
@@ -262,7 +257,7 @@ where
}
}
-fn remove_inactive_sessions<S>(states: &SessionStates<S>, now: Instant, ttl: Duration) {
+fn remove_inactive_sessions(states: &SessionStates, now: Instant, ttl: Duration) {
states.lock().retain(|_, session| {
// The registry owns one strong reference. Any others belong to active requests,
// which must keep sharing this state until their routing work finishes.
@@ -282,10 +277,7 @@ fn session_id(request: &Request) -> Option<String> {
}
#[async_trait]
-impl<S> Algorithm for FallThrough<S>
-where
- S: Default + Send + 'static,
-{
+impl Algorithm for FallThrough {
fn name(&self) -> &str {
&self.name
}
@@ -338,7 +330,7 @@ mod tests {
impl Classifier for FixedClassifier {
async fn score(
&self,
- _state: &mut (),
+ _state: &mut State,
_request: &mut Request,
_driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
@@ -386,14 +378,11 @@ mod tests {
}
/// Drives a shared router with one request, returning the completion text and selection.
- async fn run_request<S>(
- router: &Arc<FallThrough<S>>,
+ async fn run_request(
+ router: &Arc<FallThrough>,
request: Request,
serve: impl Serve,
- ) -> Result<(String, ModelId)>
- where
- S: Default + Send + 'static,
- {
+ ) -> Result<(String, ModelId)> {
let (selected_model, response) = test_drive_with_models(
router.clone(),
request,
@@ -411,13 +400,7 @@ mod tests {
}
/// Drives a shared router through one turn in the default test session.
- async fn run_turn<S>(
- router: &Arc<FallThrough<S>>,
- serve: impl Serve,
- ) -> Result<(String, ModelId)>
- where
- S: Default + Send + 'static,
- {
+ async fn run_turn(router: &Arc<FallThrough>, serve: impl Serve) -> Result<(String, ModelId)> {
run_request(router, request(), serve).await
}
@@ -436,7 +419,7 @@ mod tests {
#[async_trait]
impl Processor for Noting {
- async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
+ async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
if let Event::Decision { request, .. } = event {
prompts::append_note(request, NOTE);
}
@@ -486,7 +469,7 @@ mod tests {
// The decision came from `capable`, so the rest of that category is tried before
// the models it does not contain.
let router = Arc::new(
- FallThrough::<()>::new()
+ FallThrough::new()
.with_classifier(Arc::new(DefaultCategoryClassifier(Category::Capable))),
);
let models = crate::RuntimeModels::new(
@@ -497,7 +480,7 @@ mod tests {
.into(),
);
- let stream = router.run_stream(request(), Arc::new(models));
+ let stream = router.run_stream(request(), Arc::new(models), State::default());
tokio::pin!(stream);
while let Some(step) = stream.next().await {
if let crate::Step::Done(outcome) = step? {
@@ -515,11 +498,10 @@ mod tests {
async fn selected_target_leads_the_ordered_candidate_list() -> Result<()> {
use futures::StreamExt;
- let router =
- Arc::new(FallThrough::<()>::new().with_classifier(fixed(vec![score("mid", 0.9)])));
+ let router = Arc::new(FallThrough::new().with_classifier(fixed(vec![score("mid", 0.9)])));
let models = category_models(Category::Any, &["weak", "mid", "strong"]);
- let stream = router.run_stream(request(), Arc::new(models.into()));
+ let stream = router.run_stream(request(), Arc::new(models.into()), State::default());
tokio::pin!(stream);
while let Some(step) = stream.next().await {
if let crate::Step::Done(outcome) = step? {
@@ -537,7 +519,7 @@ mod tests {
#[tokio::test]
async fn argmax_picks_the_highest_confidence_target() -> Result<()> {
- let router = FallThrough::<()>::new()
+ let router = FallThrough::new()
.with_classifier(fixed(vec![score("weak", 0.2), score("strong", 0.9)]));
let (model, selected_model) = run_with(router, echo()).await?;
assert_eq!(model, "strong");
@@ -548,7 +530,7 @@ mod tests {
#[tokio::test]
async fn falls_through_the_first_abstaining_classifier() -> Result<()> {
// First classifier abstains (empty); the second decides.
- let router = FallThrough::<()>::new()
+ let router = FallThrough::new()
.with_classifier(fixed(vec![]))
.with_classifier(fixed(vec![score("weak", 1.0)]));
let (model, _) = run_with(router, echo()).await?;
@@ -559,7 +541,7 @@ mod tests {
#[tokio::test]
async fn first_deciding_classifier_wins_the_cascade() -> Result<()> {
// The first classifier decides; the second is never consulted.
- let router = FallThrough::<()>::new()
+ let router = FallThrough::new()
.with_classifier(fixed(vec![score("strong", 0.6)]))
.with_classifier(fixed(vec![score("weak", 1.0)]));
let (model, _) = run_with(router, echo()).await?;
@@ -569,7 +551,7 @@ mod tests {
#[tokio::test]
async fn all_abstaining_is_an_error() -> Result<()> {
- let router = FallThrough::<()>::new().with_classifier(fixed(vec![]));
+ let router = FallThrough::new().with_classifier(fixed(vec![]));
let error = run_with(router, echo())
.await
.err()
@@ -591,7 +573,7 @@ mod tests {
#[async_trait]
impl Processor for RecordingProcessor {
- async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
+ async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
let kind = match event {
Event::Request { .. } => "request",
Event::Decision { .. } => "decision",
@@ -603,7 +585,7 @@ mod tests {
}
let seen = Arc::new(Mutex::new(Vec::new()));
- let router = FallThrough::<()>::new()
+ let router = FallThrough::new()
.with_processor(Arc::new(RecordingProcessor(seen.clone())))
.with_classifier(fixed(vec![score("strong", 1.0)]));
run_with(router, echo()).await?;
@@ -621,7 +603,7 @@ mod tests {
#[async_trait]
impl Processor for Appender {
- async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
+ async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
if let Event::Request { request, .. } = event {
request
.llm_request
@@ -640,7 +622,7 @@ mod tests {
impl Classifier for TrailClassifier {
async fn score(
&self,
- _state: &mut (),
+ _state: &mut State,
request: &mut Request,
_driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
@@ -687,19 +669,26 @@ mod tests {
#[tokio::test]
async fn state_is_shared_within_a_session_and_isolated_between_sessions() -> Result<()> {
- #[derive(Default)]
- struct TurnState {
- count: u32,
+ use crate::core::state::StateValue;
+
+ fn count(state: &State) -> u32 {
+ match state.extra.get("count") {
+ Some(StateValue::Count(n)) => *n,
+ _ => 0,
+ }
}
// Increments the session turn count on every request.
struct CountingProcessor;
#[async_trait]
- impl Processor<TurnState> for CountingProcessor {
- async fn process(&self, state: &mut TurnState, event: Event<'_>) -> Result<()> {
+ impl Processor for CountingProcessor {
+ async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
if let Event::Request { .. } = event {
- state.count += 1;
+ let next = count(state) + 1;
+ state
+ .extra
+ .insert("count".to_string(), StateValue::Count(next));
}
Ok(())
}
@@ -709,20 +698,20 @@ mod tests {
struct ThresholdClassifier;
#[async_trait]
- impl Classifier<TurnState> for ThresholdClassifier {
+ impl Classifier for ThresholdClassifier {
async fn score(
&self,
- state: &mut TurnState,
+ state: &mut State,
_request: &mut Request,
_driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
- let target = if state.count >= 2 { "strong" } else { "weak" };
+ let target = if count(state) >= 2 { "strong" } else { "weak" };
Ok((Classification::Scores(vec![score(target, 1.0)]), None))
}
}
let router = Arc::new(
- FallThrough::<TurnState>::new_with_state()
+ FallThrough::new_with_state()
.with_processor(Arc::new(CountingProcessor))
.with_classifier(Arc::new(ThresholdClassifier)),
);
@@ -770,7 +759,7 @@ mod tests {
#[tokio::test]
async fn final_session_is_removed_when_routing_fails() {
- let router = Arc::new(FallThrough::<u32>::new_with_state());
+ let router = Arc::new(FallThrough::new_with_state());
let final_request = Request {
metadata: Some(Metadata {
session_id: Some("session-1".to_string()),
@@ -799,7 +788,7 @@ mod tests {
#[test]
fn cleanup_removes_only_inactive_idle_sessions() {
- let router = FallThrough::<u32>::new_with_state();
+ let router = FallThrough::new_with_state();
let _active_state = router
.session_state(&request()) // session-1
.expect("session state was inserted");
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 5aa948101..1997ab3b7 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -599,9 +599,9 @@ fn affinity_router(
/// Routes requests through a capability, escalation, or custom classifier mode.
pub struct LlmTaskClassifier {
- route: FallThrough<State>,
+ route: FallThrough,
/// Classifier used when this router is embedded in another cascade.
- inner: Arc<dyn Classifier<State>>,
+ inner: Arc<dyn Classifier>,
}
struct ClassifierRouteConfig {
@@ -615,10 +615,10 @@ struct ClassifierRouteConfig {
pub struct DefaultCategoryClassifier(pub Category);
#[async_trait]
-impl<S: Send> Classifier<S> for DefaultCategoryClassifier {
+impl Classifier for DefaultCategoryClassifier {
async fn score(
&self,
- _state: &mut S,
+ _state: &mut State,
_request: &mut Request,
driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
@@ -691,7 +691,7 @@ impl LlmTaskClassifier {
let input = TaskInput {
recent_turn_window: config.recent_turn_window,
};
- let classifier: Arc<dyn Classifier<State>> = match config.judge {
+ let classifier: Arc<dyn Classifier> = match config.judge {
CapabilityJudgeConfig::Llm(judge) => Arc::new(
JudgeClassifier::new(
StructuredJudge::new(
@@ -734,7 +734,7 @@ impl LlmTaskClassifier {
pub(crate) fn custom_classifier(
config: &CustomClassifierConfig,
fail_open: bool,
- ) -> Result<Arc<dyn Classifier<State>>> {
+ ) -> Result<Arc<dyn Classifier>> {
config.validate()?;
let contract =
ClassifierContract::from_inner_schema(&config.prompt, config.response_schema.clone())?;
@@ -772,7 +772,7 @@ impl LlmTaskClassifier {
) -> Result<Self> {
let inner = escalation::build_classifier(contract_config, config, max_output_tokens)?;
Ok(Self {
- route: FallThrough::<State>::new_with_state()
+ route: FallThrough::new_with_state()
.with_name(ALGORITHM_NAME)
.with_classifier(Arc::clone(&inner)),
inner,
@@ -785,10 +785,7 @@ impl LlmTaskClassifier {
}
/// Keeps affinity and fallback ordering identical across judge-backed modes.
- fn from_classifier(
- inner: Arc<dyn Classifier<State>>,
- config: ClassifierRouteConfig,
- ) -> Result<Self> {
+ fn from_classifier(inner: Arc<dyn Classifier>, config: ClassifierRouteConfig) -> Result<Self> {
// Only `every_request` is rejected: it retains no target, so a fallback identity has nothing to key on. Both retaining triggers can key the retained target on a message hash when the caller sends no session id.
if config.message_hash_fallback && config.classify_trigger == ClassifyTrigger::EveryRequest
{
@@ -799,7 +796,7 @@ impl LlmTaskClassifier {
});
}
// Affinity comes first so a retained assignment short-circuits the judge call.
- let mut route = FallThrough::<State>::new_with_state().with_name(ALGORITHM_NAME);
+ let mut route = FallThrough::new_with_state().with_name(ALGORITHM_NAME);
if let Some(affinity) =
affinity_router(config.classify_trigger, config.message_hash_fallback).as_ref()
{
@@ -819,7 +816,7 @@ impl LlmTaskClassifier {
}
#[async_trait]
-impl Classifier<State> for LlmTaskClassifier {
+impl Classifier for LlmTaskClassifier {
async fn score(
&self,
state: &mut State,
diff --git a/crates/libsy/src/algorithms/llm_class/decision.rs b/crates/libsy/src/algorithms/llm_class/decision.rs
index c3716bea7..35efe06a6 100644
--- a/crates/libsy/src/algorithms/llm_class/decision.rs
+++ b/crates/libsy/src/algorithms/llm_class/decision.rs
@@ -110,7 +110,7 @@ impl DecisionClassifier {
}
#[async_trait]
-impl Classifier<State> for DecisionClassifier {
+impl Classifier for DecisionClassifier {
async fn score(
&self,
_state: &mut State,
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 8b80dc14f..25a8a5ee0 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -18,6 +18,7 @@ use rand::rngs::StdRng;
use crate::algorithms::fall_through::FallThrough;
use crate::core::algorithm::{Algorithm, Driver};
use crate::core::classifier::{Classification, Classifier, Score};
+use crate::core::state::State;
use crate::{LibsyError, Result};
use switchyard_protocol::{Category, Request, Response};
@@ -77,13 +78,10 @@ fn invalid_weights(message: String) -> LibsyError {
}
#[async_trait]
-impl<S> Classifier<S> for RandomClassifier
-where
- S: Send + 'static,
-{
+impl Classifier for RandomClassifier {
async fn score(
&self,
- _state: &mut S,
+ _state: &mut State,
_request: &mut Request,
driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
@@ -122,7 +120,7 @@ where
/// Random router with fallbacks restricted to targets with positive weights.
pub struct Random {
- inner: FallThrough<()>,
+ inner: FallThrough,
classifier: Arc<RandomClassifier>,
}
@@ -130,7 +128,7 @@ impl Random {
/// Creates a random router. The models themselves will be passed at runtime.
pub fn new(weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Self> {
let classifier = Arc::new(RandomClassifier::new(weights, seed)?);
- let inner = FallThrough::<()>::new()
+ let inner = FallThrough::new()
.with_name("random")
.with_classifier(classifier.clone());
Ok(Self { inner, classifier })
@@ -296,7 +294,7 @@ mod tests {
let affinity = Arc::new(AffinityRouter::new());
let random = Arc::new(RandomClassifier::new(None, Some(42))?);
let algorithm: Arc<dyn Algorithm> = Arc::new(
- FallThrough::<()>::new()
+ FallThrough::new()
.with_name("affinity_random")
.with_processor(affinity.clone())
.with_classifier(affinity.clone())
@@ -316,7 +314,7 @@ mod tests {
.map(completion_text)
.unwrap_or_default();
- let mut state = ();
+ let mut state = State::default();
let mut request = request_for_session("session-1");
let driver = Driver::new("test", Arc::new(models.clone().into())).0;
let retained = affinity
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index 361025bd5..7e1e82c09 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -22,7 +22,7 @@ use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierCon
use super::util::prompts::prepend_system_prompt;
use super::util::stage::{
DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, Tier, fall_open_tier,
- record_decision_source, record_routing_decision,
+ record_routing_decision,
};
use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSemantics, ToolSignalProcessor};
use crate::core::algorithm::{Algorithm, Driver};
@@ -41,7 +41,7 @@ struct TierPromptProcessor {
}
#[async_trait]
-impl Processor<State> for TierPromptProcessor {
+impl Processor for TierPromptProcessor {
async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
if let Event::Decision {
request, category, ..
@@ -65,12 +65,12 @@ impl Processor<State> for TierPromptProcessor {
/// The classifiers themselves are composition-agnostic and write no state; only
/// this router knows where each sits in its cascade.
struct SourceStamp {
- inner: Arc<dyn Classifier<State>>,
+ inner: Arc<dyn Classifier>,
source: DecisionSource,
}
#[async_trait]
-impl Classifier<State> for SourceStamp {
+impl Classifier for SourceStamp {
async fn score(
&self,
state: &mut State,
@@ -80,7 +80,6 @@ impl Classifier<State> for SourceStamp {
let (classification, served) = self.inner.score(state, request, driver).await?;
// An abstaining classifier passes the turn on, so it is not its to claim.
if let Some(winner) = classification.argmax(false)? {
- record_decision_source(state, self.source);
record_routing_decision(self.source, &winner.target);
driver.set_evidence_if_empty(serde_json::json!({
"source": self.source.as_str(),
@@ -96,7 +95,7 @@ struct FallOpen {
}
#[async_trait]
-impl Classifier<State> for FallOpen {
+impl Classifier for FallOpen {
async fn score(
&self,
state: &mut State,
@@ -180,7 +179,7 @@ impl StageRouterConfig {
/// decide first, an optional capability judge takes the turns they cannot, and
/// the picker's default tier closes the cascade so a turn is never left unrouted.
pub struct StageRouter {
- route: FallThrough<State>,
+ route: FallThrough,
}
impl StageRouter {
@@ -212,7 +211,7 @@ impl Algorithm for StageRouter {
/// Wires the cascade the wrapper drives. Exposed so a composition above can
/// stack a prelude onto it.
-pub(crate) fn build_stage_route(config: StageRouterConfig) -> Result<FallThrough<State>> {
+pub(crate) fn build_stage_route(config: StageRouterConfig) -> Result<FallThrough> {
for (name, threshold) in [
("confidence_threshold", config.confidence_threshold),
("capable_first_threshold", config.capable_first_threshold),
@@ -237,7 +236,7 @@ pub(crate) fn build_stage_route(config: StageRouterConfig) -> Result<FallThrough
recent_window: config.recent_window.unwrap_or(DEFAULT_RECENT_WINDOW),
tool_semantics: config.tool_semantics,
};
- let mut router = FallThrough::<State>::new_with_state()
+ let mut router = FallThrough::new_with_state()
.with_name(STAGE_ROUTER)
.with_processor(Arc::new(signals))
.with_classifier(Arc::new(classifier));
@@ -273,17 +272,16 @@ mod tests {
use parking_lot::Mutex;
use super::*;
- use crate::algorithms::util::stage::{DECISION_SOURCE_KEY, clear_fall_open, set_fall_open};
+ use crate::algorithms::util::stage::{clear_fall_open, set_fall_open};
use crate::algorithms::util::tier_fixtures::{JUDGE, Recorder, turn_request};
- use crate::core::state::StateValue;
- use crate::core::testing::{empty_driver, test_drive_with_models};
+ use crate::core::testing::test_drive_with_models;
use switchyard_protocol::{Category, ModelId};
/// A classifier that always picks `target`, standing in for a cascade member.
struct Fixed(&'static str);
#[async_trait]
- impl Classifier<State> for Fixed {
+ impl Classifier for Fixed {
async fn score(
&self,
_state: &mut State,
@@ -305,7 +303,7 @@ mod tests {
struct Abstains;
#[async_trait]
- impl Classifier<State> for Abstains {
+ impl Classifier for Abstains {
async fn score(
&self,
_state: &mut State,
@@ -316,19 +314,29 @@ mod tests {
}
}
- async fn stamped(inner: Arc<dyn Classifier<State>>) -> Result<Option<String>> {
+ async fn stamped(inner: Arc<dyn Classifier>) -> Result<Option<String>> {
let stamp = SourceStamp {
inner,
source: DecisionSource::LlmClassifier,
};
- let mut state = State::default();
+ let (driver, mut rx) = Driver::new("test", Arc::new(crate::RuntimeModels::default()));
stamp
- .score(&mut state, &mut Request::default(), &empty_driver())
+ .score(&mut State::default(), &mut Request::default(), &driver)
.await?;
- Ok(match state.extra.get(DECISION_SOURCE_KEY) {
- Some(StateValue::String(source)) => Some(source.clone()),
- _ => None,
- })
+ driver
+ .finish(Ok(crate::RoutingOutcome::route_to(
+ "strong".into(),
+ Vec::new(),
+ Request::default(),
+ )))
+ .await?;
+ let Some(Ok(crate::Step::Done(outcome))) = rx.recv().await else {
+ panic!("expected Done step");
+ };
+ Ok(outcome
+ .metadata
+ .and_then(|metadata| metadata.evidence)
+ .and_then(|evidence| evidence.get("source")?.as_str().map(str::to_string)))
}
#[tokio::test]
@@ -440,7 +448,7 @@ mod tests {
}
#[async_trait]
- impl Processor<State> for TierDecider {
+ impl Processor for TierDecider {
async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
if matches!(event, Event::Request { .. }) {
let mut requests = self.requests.lock();
diff --git a/crates/libsy/src/algorithms/subagent.rs b/crates/libsy/src/algorithms/subagent.rs
index 74334abfd..e2a538280 100644
--- a/crates/libsy/src/algorithms/subagent.rs
+++ b/crates/libsy/src/algorithms/subagent.rs
@@ -14,13 +14,12 @@ use super::util::subagent::SubagentGate;
use crate::algorithms::llm_class::DefaultCategoryClassifier;
use crate::core::algorithm::{Algorithm, Driver};
use crate::core::classifier::Classifier;
-use crate::core::state::State;
use crate::{LibsyError, Result, RoutingOutcome};
/// Runtime components for delegated sub-agent routing.
pub struct SubagentRouterConfig {
/// Classifier invoked for delegated work according to `classify_trigger`.
- pub classifier: Arc<dyn Classifier<State>>,
+ pub classifier: Arc<dyn Classifier>,
/// Child model category used when `classifier` abstains.
pub default_target: Category,
/// Controls whether each child is classified once or on every request.
@@ -44,7 +43,7 @@ impl SubagentRouterConfig {
/// Routes delegated work independently while preserving the parent algorithm for other traffic.
pub struct SubagentRouter {
parent: Arc<dyn Algorithm>,
- subagent: FallThrough<State>,
+ subagent: FallThrough,
/// Whether the one-time warning about a harness without sub-agent identity has been emitted.
identity_warning_emitted: AtomicBool,
}
@@ -151,7 +150,7 @@ mod tests {
}
#[async_trait]
- impl Classifier<State> for ScriptedClassifier {
+ impl Classifier for ScriptedClassifier {
async fn score(
&self,
_state: &mut State,
@@ -193,7 +192,7 @@ mod tests {
}))
}
- fn configured(classifier: Arc<dyn Classifier<State>>) -> crate::Result<Arc<SubagentRouter>> {
+ fn configured(classifier: Arc<dyn Classifier>) -> crate::Result<Arc<SubagentRouter>> {
Ok(Arc::new(SubagentRouter::new(
Arc::new(Passthrough),
SubagentRouterConfig {
diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
index 6e490810d..18d2cc363 100644
--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs
+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
@@ -18,6 +18,7 @@ use super::util::subagent::SubagentOverride;
use crate::Result;
use crate::core::algorithm::Driver;
use crate::core::classifier::{Classification, Classifier, Score};
+use crate::core::state::State;
use crate::core::testing::{echo, test_drive_with_models};
use switchyard_protocol::{
Category, Metadata, ModelId, Request, Response, completion_text, slice_to_header_map,
@@ -31,7 +32,7 @@ struct AlwaysOrchestrator;
impl Classifier for AlwaysOrchestrator {
async fn score(
&self,
- _state: &mut (),
+ _state: &mut State,
_request: &mut Request,
_driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
@@ -66,7 +67,7 @@ fn request(headers: &[(&str, &str)]) -> Request {
fn router() -> Arc<FallThrough> {
let affinity = Arc::new(AffinityRouter::for_subagents());
Arc::new(
- FallThrough::<()>::new()
+ FallThrough::new()
.with_processor(affinity.clone())
.with_classifier(affinity)
.with_classifier(Arc::new(SubagentOverride::new("worker")))
@@ -145,7 +146,7 @@ async fn harness_maintenance_turns_are_not_forced_to_the_worker() -> Result<()>
/// Builds a cascade whose override scores `worker`, sharing `affinity` across instances.
fn router_overriding_to(affinity: Arc<AffinityRouter>, worker: &str) -> Arc<FallThrough> {
Arc::new(
- FallThrough::<()>::new()
+ FallThrough::new()
.with_processor(affinity.clone())
.with_classifier(affinity)
.with_classifier(Arc::new(SubagentOverride::new(worker)))
@@ -205,7 +206,7 @@ async fn a_cascade_without_the_override_still_routes_root_traffic() -> Result<()
// cascade, which is the point of composing them rather than nesting one in the other.
let affinity = Arc::new(AffinityRouter::for_subagents());
let router = Arc::new(
- FallThrough::<()>::new()
+ FallThrough::new()
.with_processor(affinity.clone())
.with_classifier(affinity)
.with_classifier(Arc::new(AlwaysOrchestrator)),
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
index 698f0a9dd..48e00ecb6 100644
--- a/crates/libsy/src/algorithms/util/affinity.rs
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -29,6 +29,7 @@ use switchyard_protocol::{Category, ContentBlock, Message, ModelId, Request, Rol
use crate::core::algorithm::{Driver, RoutingIdentity};
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::processor::{Event, Processor};
+use crate::core::state::State;
/// Upper bound on retained assignments, keeping the process-local map from growing
/// without limit; the oldest entry is evicted once the bound is reached.
@@ -169,11 +170,8 @@ impl AffinityRouter {
}
#[async_trait]
-impl<S> Processor<S> for AffinityRouter
-where
- S: Send + 'static,
-{
- async fn process(&self, _state: &mut S, event: Event<'_>) -> crate::Result<()> {
+impl Processor for AffinityRouter {
+ async fn process(&self, _state: &mut State, event: Event<'_>) -> crate::Result<()> {
if let Event::Decision {
request,
selected_model_id,
@@ -226,13 +224,10 @@ fn first_user_message_hash(request: &Request) -> Option<String> {
}
#[async_trait]
-impl<S> Classifier<S> for AffinityRouter
-where
- S: Send + 'static,
-{
+impl Classifier for AffinityRouter {
async fn score(
&self,
- _state: &mut S,
+ _state: &mut State,
request: &mut Request,
driver: &Driver,
) -> crate::Result<(Classification, Option<switchyard_protocol::Response>)> {
@@ -369,7 +364,7 @@ mod tests {
/// Folds a request and its decision through the router, retaining `model`.
async fn retain(
router: &AffinityRouter,
- state: &mut (),
+ state: &mut State,
request: &mut Request,
model: &'static str,
) -> Result<(), BoxErr> {
@@ -391,7 +386,7 @@ mod tests {
/// Scores through the definitive classification variant used by affinity.
async fn scores(
classifier: &dyn Classifier,
- state: &mut (),
+ state: &mut State,
request: &mut Request,
) -> Result<Vec<Score>, BoxErr> {
match classifier.score(state, request, &driver()).await?.0 {
@@ -403,7 +398,7 @@ mod tests {
#[tokio::test]
async fn session_retains_first_model_across_requests() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
let mut first = request(session("session-1", "agent-a"));
retain(&router, &mut state, &mut first, "model-a").await?;
@@ -420,7 +415,7 @@ mod tests {
#[tokio::test]
async fn subagent_only_retains_children_without_latching_root_traffic() -> Result<(), BoxErr> {
let router = AffinityRouter::for_subagents();
- let mut state = ();
+ let mut state = State::default();
let mut root = request(session("session-1", "root-agent"));
retain(&router, &mut state, &mut root, "model-a").await?;
@@ -440,7 +435,7 @@ mod tests {
#[tokio::test]
async fn first_decision_wins() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
let mut req = request(session("session-1", "agent-a"));
retain(&router, &mut state, &mut req, "model-a").await?;
@@ -458,7 +453,7 @@ mod tests {
#[tokio::test]
async fn subagent_is_keyed_by_agent_not_task() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
let mut first = request(subagent("child-1", "task-1"));
retain(&router, &mut state, &mut first, "model-a").await?;
@@ -476,7 +471,7 @@ mod tests {
#[tokio::test]
async fn distinct_subagents_are_assigned_independently() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
// One child in the session is pinned...
retain(
@@ -496,7 +491,7 @@ mod tests {
#[tokio::test]
async fn subagent_does_not_inherit_session_assignment() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
// The session root is pinned, but a sub-agent is keyed separately...
retain(
@@ -516,7 +511,7 @@ mod tests {
#[tokio::test]
async fn classifier_abstains_without_a_session() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
// No session id at all: nothing to key on.
let mut req = request(Metadata::default());
@@ -527,7 +522,7 @@ mod tests {
#[tokio::test]
async fn message_hash_fallback_uses_the_first_user_message() -> Result<(), BoxErr> {
let router = AffinityRouter::new().with_message_hash_fallback();
- let mut state = ();
+ let mut state = State::default();
let mut first = task_request(
None,
@@ -566,7 +561,7 @@ mod tests {
async fn subagents_only_root_traffic_does_not_warn() -> Result<(), BoxErr> {
// Abstaining on root traffic is this mode's contract, so it must not warn.
let router = AffinityRouter::for_subagents();
- let mut state = ();
+ let mut state = State::default();
let mut root = request(session("session-1", "agent-1"));
assert!(scores(&router, &mut state, &mut root).await?.is_empty());
@@ -611,7 +606,7 @@ mod tests {
#[tokio::test]
async fn metadata_session_takes_precedence_over_message_hash() -> Result<(), BoxErr> {
let router = AffinityRouter::new().with_message_hash_fallback();
- let mut state = ();
+ let mut state = State::default();
let mut first = task_request(
Some(session("session-1", "agent-a")),
@@ -637,7 +632,7 @@ mod tests {
async fn subagent_without_a_session_abstains_and_warns() -> Result<(), BoxErr> {
for session_id in [None, Some(String::new())] {
let router = AffinityRouter::new().with_message_hash_fallback();
- let mut state = ();
+ let mut state = State::default();
let mut subagent = task_request(
Some(Metadata {
session_id,
@@ -666,7 +661,7 @@ mod tests {
let router = Arc::new(AffinityRouter::new());
let processor: Arc<dyn Processor> = router.clone();
let classifier: Arc<dyn Classifier> = router;
- let mut state = ();
+ let mut state = State::default();
let mut first = request(session("session-1", "agent-a"));
processor
@@ -693,7 +688,7 @@ mod tests {
#[tokio::test]
async fn decision_without_an_affinity_identity_is_ignored() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
let mut unkeyed = request(Metadata::default());
router
@@ -716,7 +711,7 @@ mod tests {
#[tokio::test]
async fn decisions_retain_their_originating_request_identity() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
let mut first = request(session("session-1", "agent-a"));
let mut second = request(session("session-2", "agent-b"));
@@ -761,7 +756,7 @@ mod tests {
#[tokio::test]
async fn distinct_sessions_are_assigned_independently() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
retain(
&router,
@@ -804,7 +799,7 @@ mod tests {
#[tokio::test]
async fn subagent_without_an_agent_id_abstains_and_warns() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
// The sub-agent flag is set but no agent id is present, so no key can be formed;
// the request is neither retained nor scored.
@@ -826,7 +821,7 @@ mod tests {
#[tokio::test]
async fn assignments_are_bounded_by_the_cap() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
// One distinct session past the cap forces exactly one eviction.
for index in 0..=MAX_ASSIGNMENTS {
@@ -849,7 +844,7 @@ mod tests {
async fn release_on_user_turn_drops_the_assignment_only_when_the_user_speaks()
-> Result<(), BoxErr> {
let router = AffinityRouter::new().with_release_on_user_turn();
- let mut state = ();
+ let mut state = State::default();
let mut opening = task_request(Some(session("session-1", "agent-a")), "add caching", None);
retain(&router, &mut state, &mut opening, "weak").await?;
@@ -884,7 +879,7 @@ mod tests {
#[tokio::test]
async fn latch_only_retains_matching_models() -> Result<(), BoxErr> {
let router = AffinityRouter::new().with_latch_only(["strong"]);
- let mut state = ();
+ let mut state = State::default();
let mut req = request(session("session-1", "agent-a"));
// A "weak" decision is not retained — a later turn is not latched.
@@ -906,7 +901,7 @@ mod tests {
#[tokio::test]
async fn an_unprovisioned_any_group_does_not_evict_assignments() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = ();
+ let mut state = State::default();
let mut req = request(session("session-1", "agent-a"));
retain(&router, &mut state, &mut req, "model-a").await?;
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 2ca10cc82..f59124a62 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -344,7 +344,7 @@ fn client_error_reason(error: &LlmClientError) -> &'static str {
}
#[async_trait]
-impl<J, P> Classifier<State> for JudgeClassifier<J, P>
+impl<J, P> Classifier for JudgeClassifier<J, P>
where
J: Judge,
P: JudgePolicy<Verdict = J::Verdict>,
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index 3453a7f9f..40fd2c5d8 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -125,9 +125,6 @@ impl PickerMode {
}
}
-/// `State.extra` key under which the turn's [`DecisionSource`] is recorded.
-pub const DECISION_SOURCE_KEY: &str = "decision_source";
-
/// `State.extra` key under which a default-tier override is recorded.
const FALL_OPEN_KEY: &str = "fall_open";
@@ -154,20 +151,7 @@ pub(crate) fn fall_open_tier(state: &State) -> Option<Tier> {
}
}
-/// Record which component decided the turn.
-pub(crate) fn record_decision_source(state: &mut State, source: DecisionSource) {
- state.extra.insert(
- DECISION_SOURCE_KEY.to_string(),
- StateValue::String(source.as_str().to_string()),
- );
-}
-
/// What produced a decision — for stats and explainability.
-///
-/// Each is stamped by the component that knows it, so a turn's final label names
-/// whoever actually decided it. [`Ambiguous`](Self::Ambiguous) is the exception:
-/// the signal scorer records it on its way out, and whichever classifier resolves
-/// the turn overwrites it.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum DecisionSource {
/// Hard override (critical severity or context compaction).
@@ -577,8 +561,7 @@ impl StageClassifier {
/// The signals could not decide, so this turn belongs to whatever the cascade
/// has behind this classifier.
- fn abstain(state: &mut State) -> Classification {
- record_decision_source(state, DecisionSource::Ambiguous);
+ fn abstain() -> Classification {
Classification::Ambiguous(Vec::new())
}
@@ -629,7 +612,7 @@ impl StageClassifier {
}
#[async_trait]
-impl Classifier<State> for StageClassifier {
+impl Classifier for StageClassifier {
async fn score(
&self,
state: &mut State,
@@ -639,7 +622,7 @@ impl Classifier<State> for StageClassifier {
let Some(signal) = state.tool_signals.clone() else {
// No tool activity yet — nothing to score, so the signals have no
// opinion, same as a below-threshold turn.
- return Ok((Self::abstain(state), None));
+ return Ok((Self::abstain(), None));
};
let capable_hold_key = Self::capable_hold_key(request);
@@ -678,7 +661,6 @@ impl Classifier<State> for StageClassifier {
{
self.set_capable_hold(state, &capable_hold_key);
}
- record_decision_source(state, source);
record_routing_decision(source, target);
if source == DecisionSource::Override
&& let Some(reason) = override_reason(&signal)
@@ -715,7 +697,7 @@ impl Classifier<State> for StageClassifier {
None,
))
}
- PickOutcome::ConsultClassifier { .. } => Ok((Self::abstain(state), None)),
+ PickOutcome::ConsultClassifier { .. } => Ok((Self::abstain(), None)),
}
}
}
@@ -1039,10 +1021,6 @@ mod tests {
.score(&mut state, &mut Request::default(), &driver())
.await?;
assert!(classification.0.argmax(false)?.is_none());
- assert!(matches!(
- state.extra.get(DECISION_SOURCE_KEY),
- Some(StateValue::String(source)) if source == "ambiguous"
- ));
Ok(())
}
@@ -1067,11 +1045,6 @@ mod tests {
}
_ => panic!("expected a definite classification"),
}
- // The decision source travels downstream (for handoff-note gating).
- assert!(matches!(
- state.extra.get(DECISION_SOURCE_KEY),
- Some(StateValue::String(source)) if source == "override"
- ));
Ok(())
}
@@ -1093,10 +1066,6 @@ mod tests {
Classification::Scores(scores) => assert_eq!(scores[0].target, "weak"),
_ => panic!("expected a definite classification"),
}
- assert!(matches!(
- state.extra.get(DECISION_SOURCE_KEY),
- Some(StateValue::String(source)) if source == "dimensions"
- ));
Ok(())
}
@@ -1196,17 +1165,12 @@ mod tests {
#[tokio::test]
async fn classifier_falls_open_to_default_and_records_it() -> Result<()> {
- // A quiet signal corroborates neither axis, so the scorer abstains and
- // records why.
+ // A quiet signal corroborates neither axis, so the scorer abstains.
let mut state = state_with(ToolSignals::default());
let classification = StageClassifier::new(PickerMode::EfficientFirst, 0.5)
.score(&mut state, &mut Request::default(), &driver())
.await?;
assert!(classification.0.argmax(false)?.is_none());
- assert!(matches!(
- state.extra.get(DECISION_SOURCE_KEY),
- Some(StateValue::String(source)) if source == "ambiguous"
- ));
Ok(())
}
diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rs
index d3783d551..12b5692e3 100644
--- a/crates/libsy/src/algorithms/util/subagent.rs
+++ b/crates/libsy/src/algorithms/util/subagent.rs
@@ -22,6 +22,7 @@ use async_trait::async_trait;
use crate::Result;
use crate::core::algorithm::Driver;
use crate::core::classifier::{Classification, Classifier, Score};
+use crate::core::state::State;
use switchyard_protocol::{
ContentBlock, LlmRequest, Message, Metadata, ModelId, Request, Response, Role,
};
@@ -30,13 +31,13 @@ use switchyard_protocol::{
///
/// The inner classifier receives a prompt-only request clone. The original request remains
/// unchanged for the selected child model.
-pub struct SubagentGate<S> {
- inner: Arc<dyn Classifier<S>>,
+pub struct SubagentGate {
+ inner: Arc<dyn Classifier>,
}
-impl<S> SubagentGate<S> {
+impl SubagentGate {
/// Wraps `inner` with delegated-work detection.
- pub fn new(inner: Arc<dyn Classifier<S>>) -> Self {
+ pub fn new(inner: Arc<dyn Classifier>) -> Self {
Self { inner }
}
}
@@ -71,13 +72,10 @@ fn delegated_prompt_request(request: &Request) -> Option<Request> {
}
#[async_trait]
-impl<S> Classifier<S> for SubagentGate<S>
-where
- S: Send + 'static,
-{
+impl Classifier for SubagentGate {
async fn score(
&self,
- state: &mut S,
+ state: &mut State,
request: &mut Request,
driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
@@ -116,13 +114,10 @@ impl SubagentOverride {
}
#[async_trait]
-impl<S> Classifier<S> for SubagentOverride
-where
- S: Send + 'static,
-{
+impl Classifier for SubagentOverride {
async fn score(
&self,
- _state: &mut S,
+ _state: &mut State,
request: &mut Request,
driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
@@ -168,7 +163,7 @@ mod tests {
/// Scores `headers` through the override, returning the winning target if it scored.
async fn selected(headers: &[(&str, &str)]) -> Result<Option<ModelId>> {
- let mut state = ();
+ let mut state = State::default();
let classification = SubagentOverride::new("worker")
.score(&mut state, &mut request(headers), &empty_driver())
.await?;
@@ -216,7 +211,7 @@ mod tests {
async fn delegated_work_is_scored_definitively() -> Result<()> {
// Confidence 1.0 under `Scores` (never `Ambiguous`), so the cascade stops here
// rather than consulting later classifiers.
- let mut state = ();
+ let mut state = State::default();
let classification = SubagentOverride::new("worker")
.score(
&mut state,
diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs
index 914a87aff..e3a5b6274 100644
--- a/crates/libsy/src/algorithms/util/tool_signals.rs
+++ b/crates/libsy/src/algorithms/util/tool_signals.rs
@@ -474,7 +474,7 @@ impl Default for ToolSignalProcessor {
}
#[async_trait]
-impl Processor<State> for ToolSignalProcessor {
+impl Processor for ToolSignalProcessor {
async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
if let Event::Request { request: req, .. } = event {
let tool_signal = ToolSignals::from_request_with_semantics(
diff --git a/crates/libsy/src/algorithms/vgr.rs b/crates/libsy/src/algorithms/vgr.rs
index 0244ad297..3509dc2f2 100644
--- a/crates/libsy/src/algorithms/vgr.rs
+++ b/crates/libsy/src/algorithms/vgr.rs
@@ -36,7 +36,7 @@ mod tests;
/// A verification-gated route between a local and a capable tier.
pub struct Vgr {
- route: FallThrough<runtime::VgrState>,
+ route: FallThrough,
}
impl Vgr {
diff --git a/crates/libsy/src/algorithms/vgr/runtime.rs b/crates/libsy/src/algorithms/vgr/runtime.rs
index 51af4121f..bd248cfcf 100644
--- a/crates/libsy/src/algorithms/vgr/runtime.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime.rs
@@ -25,15 +25,42 @@ use crate::algorithms::util::decisive;
use crate::algorithms::util::prompts::append_note;
use crate::core::algorithm::Driver;
use crate::core::classifier::{Classification, Classifier};
+use crate::core::state::{State, StateValue};
use crate::{LibsyError, Result};
-#[derive(Default)]
-pub(super) struct VgrState {
- pub(super) turn_streak: u32,
- pub(super) turn_latched: bool,
- pub(super) turn_started: Option<u32>,
- pub(super) handoff_sent: bool,
- pub(super) local_served: bool,
+/// Consecutive escalation votes. Reaching [`TURN_CONFIRMATIONS`] latches the session to the
+/// capable tier; once latched, verification is skipped, so the count never resets.
+pub(super) const TURN_STREAK_KEY: &str = "vgr_turn_streak";
+/// Unix seconds at the first request of the current user turn.
+const TURN_STARTED_KEY: &str = "vgr_turn_started";
+/// Set once the handoff note was sent in the current user turn.
+const HANDOFF_SENT_KEY: &str = "vgr_handoff_sent";
+/// Set once the local tier served a response since the last handoff.
+pub(super) const LOCAL_SERVED_KEY: &str = "vgr_local_served";
+
+fn count(state: &State, key: &str) -> Option<u32> {
+ match state.extra.get(key) {
+ Some(StateValue::Count(n)) => Some(*n),
+ _ => None,
+ }
+}
+
+fn set_count(state: &mut State, key: &str, value: u32) {
+ state
+ .extra
+ .insert(key.to_string(), StateValue::Count(value));
+}
+
+pub(super) fn set_flag(state: &mut State, key: &str, on: bool) {
+ state.extra.insert(key.to_string(), StateValue::Bool(on));
+}
+
+fn flag(state: &State, key: &str) -> bool {
+ matches!(state.extra.get(key), Some(StateValue::Bool(true)))
+}
+
+pub(super) fn turn_streak(state: &State) -> u32 {
+ count(state, TURN_STREAK_KEY).unwrap_or(0)
}
struct RequestContext<'a> {
@@ -43,7 +70,7 @@ struct RequestContext<'a> {
}
/// Consecutive escalation votes that latch a session to the capable tier.
-const TURN_CONFIRMATIONS: u32 = 2;
+pub(super) const TURN_CONFIRMATIONS: u32 = 2;
const TURN_ESCALATE_AT: f64 = 0.5;
type Scored = (Classification, Option<Response>);
@@ -54,18 +81,18 @@ pub(super) struct VgrClassifier {
}
#[async_trait]
-impl Classifier<VgrState> for VgrClassifier {
+impl Classifier for VgrClassifier {
async fn score(
&self,
- state: &mut VgrState,
+ state: &mut State,
request: &mut Request,
driver: &Driver,
) -> Result<Scored> {
let started = Instant::now();
let local = &self.config.targets.local;
if has_new_user_turn(&request.llm_request.messages) {
- state.turn_started = None;
- state.handoff_sent = false;
+ state.extra.remove(TURN_STARTED_KEY);
+ set_flag(state, HANDOFF_SENT_KEY, false);
}
let turn_budget_spent = self.turn_budget_spent(state);
let short_circuit = if self.config.mode == ServingMode::Off {
@@ -77,7 +104,7 @@ impl Classifier<VgrState> for VgrClassifier {
.is_some_and(|switch| switch.is_engaged())
{
Some("kill_switch")
- } else if state.turn_latched {
+ } else if turn_streak(state) >= TURN_CONFIRMATIONS {
Some("turn_verification_latched")
} else if turn_budget_spent {
Some("local_turn_budget")
@@ -160,7 +187,7 @@ impl Classifier<VgrState> for VgrClassifier {
));
}
log(driver, "turn_verification_complete", None, Route::Local);
- state.local_served = true;
+ set_flag(state, LOCAL_SERVED_KEY, true);
return Ok((decisive(local), Some(buffered.into_response())));
}
@@ -180,7 +207,7 @@ impl Classifier<VgrState> for VgrClassifier {
));
}
log(driver, "decided", Some(caps.branch), Route::Local);
- state.local_served = true;
+ set_flag(state, LOCAL_SERVED_KEY, true);
Ok((decisive(local), Some(buffered.into_response())))
}
}
@@ -192,7 +219,7 @@ impl VgrClassifier {
reason: &'static str,
branch: Option<Branch>,
request: &mut Request,
- state: &mut VgrState,
+ state: &mut State,
unverified_final: Option<&str>,
) -> Scored {
if reason != "mode_off" {
@@ -207,33 +234,29 @@ impl VgrClassifier {
/// The clock starts at the first request of the user turn and includes the
/// client's tool execution, since that is the time the client's own limit
/// is spending.
- fn turn_budget_spent(&self, state: &mut VgrState) -> bool {
+ fn turn_budget_spent(&self, state: &mut State) -> bool {
let Some(budget) = self.config.local_turn_budget else {
return false;
};
let now = unix_seconds();
- let started = *state.turn_started.get_or_insert(now);
+ let started = count(state, TURN_STARTED_KEY).unwrap_or(now);
+ set_count(state, TURN_STARTED_KEY, started);
u64::from(now.saturating_sub(started)) >= budget.as_secs()
}
/// Tells the capable tier it is inheriting unverified tool-using work.
///
/// Sent at most once per user turn, on its first escalation.
- fn hand_off(
- &self,
- request: &mut Request,
- state: &mut VgrState,
- unverified_final: Option<&str>,
- ) {
+ fn hand_off(&self, request: &mut Request, state: &mut State, unverified_final: Option<&str>) {
if !self.config.agentic_handoff
- || state.handoff_sent
- || !state.local_served
+ || flag(state, HANDOFF_SENT_KEY)
+ || !flag(state, LOCAL_SERVED_KEY)
|| !text::has_tool_trajectory(request)
{
return;
}
- state.handoff_sent = true;
- state.local_served = false;
+ set_flag(state, HANDOFF_SENT_KEY, true);
+ set_flag(state, LOCAL_SERVED_KEY, false);
let note = handoff_note(unverified_final);
append_note(request, &note);
}
@@ -243,7 +266,7 @@ impl VgrClassifier {
&self,
context: &RequestContext<'_>,
proposed: &AggLlmResponse,
- state: &mut VgrState,
+ state: &mut State,
) -> bool {
let mut call = rungs::build_request(
Question::TurnTrajectory,
@@ -260,7 +283,7 @@ impl VgrClassifier {
Some(Err(error)) => {
if *judge == self.config.targets.local && endpoint_failure(&error) {
self.breaker.failure();
- state.turn_latched = true;
+ set_count(state, TURN_STREAK_KEY, TURN_CONFIRMATIONS);
return true;
}
return false;
@@ -270,14 +293,11 @@ impl VgrClassifier {
return false;
};
if probability < TURN_ESCALATE_AT {
- state.turn_streak = 0;
+ set_count(state, TURN_STREAK_KEY, 0);
return false;
}
- state.turn_streak = state.turn_streak.saturating_add(1);
- let streak = state.turn_streak;
- if streak >= TURN_CONFIRMATIONS {
- state.turn_latched = true;
- }
+ let streak = turn_streak(state).saturating_add(1);
+ set_count(state, TURN_STREAK_KEY, streak);
streak >= TURN_CONFIRMATIONS
}
diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
index 8bdd7f4bd..f037dcff5 100644
--- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs
+++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs
@@ -15,6 +15,7 @@ use super::super::tests::{call, request, result};
use crate::Result;
use crate::algorithms::vgr::Vgr;
use crate::core::algorithm::Algorithm;
+use crate::core::state::{State, StateValue};
use crate::core::testing::{category_models, reply, test_drive_with_models};
fn route(configure: impl FnOnce(&mut VgrConfig)) -> Result<Arc<dyn Algorithm>> {
@@ -271,7 +272,7 @@ async fn verifier_budget_exhaustion_is_not_an_endpoint_failure() -> Result<()> {
breaker: super::CircuitBreaker::new(config.breaker),
config,
};
- let mut state = super::VgrState::default();
+ let mut state = State::default();
let task = session(vec![Message::text(Role::User, "list files")]);
let LlmResponse::Agg(proposed) = complete_call().llm_response else {
unreachable!();
@@ -292,7 +293,7 @@ async fn verifier_budget_exhaustion_is_not_an_endpoint_failure() -> Result<()> {
let escalates = judging.await;
drop(pending_call);
assert!(!escalates);
- assert!(!state.turn_latched);
+ assert!(super::turn_streak(&state) < super::TURN_CONFIRMATIONS);
assert!(classifier.breaker.attempt().is_some());
Ok(())
}
@@ -305,7 +306,7 @@ async fn skipped_requests_leave_the_recovery_trial_available() -> Result<()> {
for skip in ["image", "latched", "off", "kill", "deadline"] {
let mut classifier = recovering_classifier();
- let mut state = super::VgrState::default();
+ let mut state = State::default();
let mut task = request(vec![Message::text(Role::User, "task")]);
match skip {
"image" => {
@@ -316,7 +317,12 @@ async fn skipped_requests_leave_the_recovery_trial_available() -> Result<()> {
},
}]
}
- "latched" => state.turn_latched = true,
+ "latched" => {
+ state.extra.insert(
+ super::TURN_STREAK_KEY.to_string(),
+ StateValue::Count(super::TURN_CONFIRMATIONS),
+ );
+ }
"off" => classifier.config.mode = ServingMode::Off,
"kill" => {
let switch = super::super::KillSwitch::new();
@@ -343,7 +349,7 @@ async fn cancelling_local_generation_releases_the_recovery_trial() -> Result<()>
use crate::core::classifier::Classifier;
let classifier = recovering_classifier();
- let mut state = super::VgrState::default();
+ let mut state = State::default();
let mut task = request(vec![Message::text(Role::User, "task")]);
let (driver, _steps) = Driver::new("vgr", Arc::new(RuntimeModels::default()));
let mut pending = classifier.score(&mut state, &mut task, &driver);
@@ -592,10 +598,8 @@ fn handoff_preserves_existing_messages_and_non_text_content() {
});
}
let original = task.llm_request.messages.clone();
- let mut state = super::VgrState {
- local_served: true,
- ..Default::default()
- };
+ let mut state = State::default();
+ super::set_flag(&mut state, super::LOCAL_SERVED_KEY, true);
classifier.hand_off(&mut task, &mut state, Some("unchecked result"));
// A notice may be appended, but all original content stays unchanged.
let messages = &task.llm_request.messages;
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 6e45c1267..a43a9edda 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -28,7 +28,7 @@ use switchyard_protocol::{
Category, DecisionRequest, DecisionResponse, ModelId, Request, Response,
};
-use crate::{DriverError, LibsyError, Result, observability};
+use crate::{DriverError, LibsyError, Result, State, observability};
/// A boxed, `Send` stream of [`Step`]s — the output of
/// [`Algorithm::run_stream`]. Boxed so the trait method that produces it keeps
@@ -265,6 +265,8 @@ pub struct RoutingOutcome {
/// Constructors leave this empty; [`Algorithm::run_stream`] fills it before publishing a
/// successful outcome.
pub metadata: Option<crate::OutcomeMetadata>,
+ /// Routing state after the run, for hosts that store state outside the algorithm.
+ pub state: State,
}
impl RoutingOutcome {
@@ -291,6 +293,7 @@ impl RoutingOutcome {
request,
response: None,
metadata: None,
+ state: State::default(),
}
}
@@ -303,6 +306,7 @@ impl RoutingOutcome {
request,
response: Some(response),
metadata: None,
+ state: State::default(),
}
}
}
@@ -318,6 +322,9 @@ pub struct Driver {
/// Run-scoped evidence shared by driver clones and attached only to a successful outcome.
evidence: Arc<Mutex<Option<Value>>>,
+ /// Caller-supplied routing state, read and updated when a request carries no session ID.
+ state: Arc<Mutex<State>>,
+
/// Every group this run may route over, shared by all driver clones.
models: Arc<RuntimeModels>,
@@ -342,6 +349,7 @@ impl Driver {
step_tx,
algorithm: algorithm.to_string(),
evidence: Arc::new(Mutex::new(None)),
+ state: Arc::new(Mutex::new(State::default())),
models,
scope: Scope::Parent,
},
@@ -349,6 +357,14 @@ impl Driver {
)
}
+ pub(crate) fn take_state(&self) -> State {
+ std::mem::take(&mut *self.state.lock())
+ }
+
+ pub(crate) fn set_state(&self, state: State) {
+ *self.state.lock() = state;
+ }
+
/// Replace the current run's evidence when a component makes the final decision.
pub(crate) fn set_evidence(&self, evidence: Value) {
*self.evidence.lock() = Some(evidence);
@@ -501,6 +517,7 @@ impl Driver {
/// when the algorithm finishes.
pub(crate) async fn finish(&self, result: Result<RoutingOutcome>) -> Result<()> {
let result = result.map(|mut outcome| {
+ outcome.state = self.take_state();
let metadata = outcome.metadata.get_or_insert_with(|| {
crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take())
});
@@ -563,7 +580,22 @@ where
F: Fn(Call) -> Fut,
Fut: Future<Output = Result<()>>,
{
- let stream = algorithm.run_stream(request, models);
+ drive_with_state(algorithm, request, models, State::default(), serve).await
+}
+
+/// [`drive`] starting from caller-supplied `state`.
+pub(crate) async fn drive_with_state<F, Fut>(
+ algorithm: Arc<dyn Algorithm>,
+ request: Request,
+ models: Arc<RuntimeModels>,
+ state: State,
+ serve: F,
+) -> Result<RoutingOutcome>
+where
+ F: Fn(Call) -> Fut,
+ Fut: Future<Output = Result<()>>,
+{
+ let stream = algorithm.run_stream(request, models, state);
tokio::pin!(stream);
let mut in_flight = futures::stream::FuturesUnordered::new();
@@ -709,8 +741,14 @@ pub trait Algorithm: Send + Sync + 'static {
/// the stream aborts the spawned algorithm task.
///
/// Every invocation owns a separate [`Driver`].
- fn run_stream(self: Arc<Self>, request: Request, models: Arc<RuntimeModels>) -> StepStream {
+ fn run_stream(
+ self: Arc<Self>,
+ request: Request,
+ models: Arc<RuntimeModels>,
+ state: State,
+ ) -> StepStream {
let (driver, step_rx) = Driver::new(self.name(), models);
+ driver.set_state(state);
let span = observability::run_span(self.name(), &request);
let handle = tokio::spawn(
async move {
@@ -1166,8 +1204,11 @@ mod tests {
async fn run_offloads_via_promise_then_finishes() -> Result<()> {
// Every call is offloaded via a promise the orchestrator surfaces as a
// `CallModel` step for us to fulfill.
- let stream = orch(target_set(&["offload/model"]))
- .run_stream(request(), Arc::new(RuntimeModels::default()));
+ let stream = orch(target_set(&["offload/model"])).run_stream(
+ request(),
+ Arc::new(RuntimeModels::default()),
+ State::default(),
+ );
tokio::pin!(stream);
let mut saw_call = false;
@@ -1282,8 +1323,11 @@ mod tests {
// A client-less target offloads its call; we fulfill the promise with an
// Err, which must flow back through `call_model_target` into the algorithm and
// out as an error step — not a response.
- let stream = orch(target_set(&["offload/model"]))
- .run_stream(request(), Arc::new(RuntimeModels::default()));
+ let stream = orch(target_set(&["offload/model"])).run_stream(
+ request(),
+ Arc::new(RuntimeModels::default()),
+ State::default(),
+ );
tokio::pin!(stream);
let mut saw_error = false;
@@ -1360,7 +1404,11 @@ mod tests {
dropped: dropped.clone(),
});
- let stream = algo.run_stream(request(), Arc::new(RuntimeModels::default()));
+ let stream = algo.run_stream(
+ request(),
+ Arc::new(RuntimeModels::default()),
+ State::default(),
+ );
started_rx
.recv()
.await
@@ -1397,7 +1445,11 @@ mod tests {
}
let algo: Arc<dyn Algorithm> = Arc::new(Panicky);
- let stream = algo.run_stream(request(), Arc::new(RuntimeModels::default()));
+ let stream = algo.run_stream(
+ request(),
+ Arc::new(RuntimeModels::default()),
+ State::default(),
+ );
tokio::pin!(stream);
let mut saw_error = false;
diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rs
index ceacc8b3c..06acfae3a 100644
--- a/crates/libsy/src/core/classifier.rs
+++ b/crates/libsy/src/core/classifier.rs
@@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
use crate::core::algorithm::Driver;
+use crate::core::state::State;
use crate::{LibsyError, Result};
use async_trait::async_trait;
use switchyard_protocol::{Category, ModelId, Request, Response};
@@ -74,7 +75,7 @@ fn argmax(scores: &[Score]) -> Result<Option<Score>> {
/// Scores targets from the current request and the composition's state.
#[async_trait]
-pub trait Classifier<S = ()>: Send + Sync {
+pub trait Classifier: Send + Sync {
/// Score the classifier's targets given the current state and request.
///
/// `driver` lets a classifier inspect runtime models and offload model calls.
@@ -86,7 +87,7 @@ pub trait Classifier<S = ()>: Send + Sync {
/// only read it.
async fn score(
&self,
- state: &mut S,
+ state: &mut State,
request: &mut Request,
driver: &Driver,
) -> Result<(Classification, Option<Response>)>;
@@ -180,14 +181,16 @@ mod tests {
struct RecordingClassifier;
#[async_trait]
- impl Classifier<bool> for RecordingClassifier {
+ impl Classifier for RecordingClassifier {
async fn score(
&self,
- state: &mut bool,
+ state: &mut State,
request: &mut Request,
_driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
- *state = true;
+ state
+ .extra
+ .insert("ran".to_string(), crate::core::state::StateValue::Count(1));
let target = request.model_id().unwrap_or(ModelId::from("auto"));
Ok((
Classification::Scores(vec![Score {
@@ -202,7 +205,7 @@ mod tests {
#[tokio::test]
async fn classifier_reads_request_and_mutates_state() -> Result<()> {
- let mut state = false;
+ let mut state = State::default();
let mut request = Request {
llm_request: text_request(Some("strong".to_string()), "hi"),
raw_request: None,
@@ -215,7 +218,10 @@ mod tests {
classification.argmax(false)?.map(|s| s.target),
Some(ModelId::from("strong"))
);
- assert!(state);
+ assert_eq!(
+ state.extra.get("ran"),
+ Some(&crate::core::state::StateValue::Count(1))
+ );
Ok(())
}
@@ -226,7 +232,7 @@ mod tests {
impl Classifier for RewritingClassifier {
async fn score(
&self,
- _state: &mut (),
+ _state: &mut State,
request: &mut Request,
_driver: &Driver,
) -> Result<(Classification, Option<Response>)> {
@@ -244,7 +250,7 @@ mod tests {
#[tokio::test]
async fn classifier_rewrites_the_request_in_place() -> Result<()> {
- let mut state = ();
+ let mut state = State::default();
let mut request = Request {
llm_request: text_request(Some("auto".to_string()), "hi"),
raw_request: None,
diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rs
index a94c047d3..b2207d4ab 100644
--- a/crates/libsy/src/core/processor.rs
+++ b/crates/libsy/src/core/processor.rs
@@ -3,6 +3,7 @@
use crate::Result;
use crate::core::algorithm::Driver;
+use crate::core::state::State;
use async_trait::async_trait;
use switchyard_protocol::{AggLlmResponse, Category, ModelId, Request};
@@ -40,21 +41,19 @@ pub enum Event<'a> {
/// Collects events as the algorithm runs and mutates the composition's state.
#[async_trait]
-pub trait Processor<S = ()>: Send + Sync {
+pub trait Processor: Send + Sync {
/// Process an event, accumulating facts into `state`. Request-bearing events
/// ([`Event::Request`], [`Event::Decision`]) may also be rewritten in place.
- async fn process(&self, state: &mut S, event: Event<'_>) -> Result<()>;
+ async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()>;
}
#[cfg(test)]
mod tests {
use super::*;
+ use crate::core::state::StateValue;
use crate::core::testing::empty_driver;
- use std::collections::HashMap;
use switchyard_protocol::{text_request, text_response};
- type TestState = HashMap<&'static str, u32>;
-
/// The key each event variant tallies under.
fn event_key(event: &Event<'_>) -> &'static str {
match event {
@@ -65,17 +64,22 @@ mod tests {
}
/// Reads a count, treating a missing key as zero.
- fn count(state: &TestState, key: &'static str) -> u32 {
- state.get(key).copied().unwrap_or_default()
+ fn count(state: &State, key: &'static str) -> u32 {
+ match state.extra.get(key) {
+ Some(StateValue::Count(n)) => *n,
+ _ => 0,
+ }
}
/// Tallies each event variant under its own key.
struct CountingProcessor;
#[async_trait]
- impl Processor<TestState> for CountingProcessor {
- async fn process(&self, state: &mut TestState, event: Event<'_>) -> Result<()> {
- *state.entry(event_key(&event)).or_default() += 1;
+ impl Processor for CountingProcessor {
+ async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
+ let key = event_key(&event);
+ let next = count(state, key) + 1;
+ state.extra.insert(key.to_string(), StateValue::Count(next));
Ok(())
}
}
@@ -91,7 +95,7 @@ mod tests {
#[tokio::test]
async fn processor_tallies_each_event_variant_into_state() -> Result<()> {
let processor = CountingProcessor;
- let mut state = TestState::default();
+ let mut state = State::default();
let mut req = request();
let response = text_response(None, "ok");
let selected_model_id = ModelId::from("test/model");
@@ -128,7 +132,7 @@ mod tests {
#[tokio::test]
async fn process_accumulates_state_across_repeated_events() -> Result<()> {
let processor = CountingProcessor;
- let mut state = TestState::default();
+ let mut state = State::default();
let mut req = request();
for _ in 0..3 {
@@ -152,7 +156,7 @@ mod tests {
#[async_trait]
impl Processor for RewritingProcessor {
- async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
+ async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
match event {
Event::Request { request, .. } | Event::Decision { request, .. } => {
request.llm_request.model = Some("rewritten".to_string());
@@ -165,7 +169,7 @@ mod tests {
#[tokio::test]
async fn processor_rewrites_the_request_in_place() -> Result<()> {
- let mut state = ();
+ let mut state = State::default();
let mut req = request();
assert_eq!(req.model_id(), Some("auto".into()));
diff --git a/crates/libsy/src/core/state.rs b/crates/libsy/src/core/state.rs
index 01ccb08a9..b21416d99 100644
--- a/crates/libsy/src/core/state.rs
+++ b/crates/libsy/src/core/state.rs
@@ -3,32 +3,73 @@
//! Built-in state shared by processors and classifiers across a session.
-use std::collections::HashMap;
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
use crate::algorithms::util::tool_signals::ToolSignals;
/// A value in a session's [`State`].
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
pub enum StateValue {
/// Text value.
String(String),
/// Nonnegative counter.
Count(u32),
- /// Signed integer value.
- Int(i32),
- /// Floating-point value.
- Scalar(f32),
+ /// Flag value.
+ Bool(bool),
}
/// Routing facts accumulated across one session's algorithm runs.
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct State {
- /// Number of turns processed for this session.
- pub turn_count: u32,
/// Tool-result signals for the current request, set by the tool-signal
/// processor. `None` until it runs or when the request has no tool activity,
/// so routers must treat absence as "no signal yet".
+ #[serde(skip)]
pub tool_signals: Option<ToolSignals>,
/// Algorithm-specific state keyed by stable internal names.
- pub extra: HashMap<String, StateValue>,
+ #[serde(flatten)]
+ pub extra: BTreeMap<String, StateValue>,
+}
+
+impl PartialEq for State {
+ fn eq(&self, other: &Self) -> bool {
+ self.extra == other.extra
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn state_round_trips_through_json_and_compares_extra_only() -> serde_json::Result<()> {
+ let mut state = State::default();
+ state
+ .extra
+ .insert("capable_hold_turns".to_string(), StateValue::Count(2));
+ state.extra.insert(
+ "fall_open".to_string(),
+ StateValue::String("capable".to_string()),
+ );
+ state
+ .extra
+ .insert("vgr_handoff_sent".to_string(), StateValue::Bool(true));
+ let json = serde_json::json!({
+ "capable_hold_turns": {"count": 2},
+ "fall_open": {"string": "capable"},
+ "vgr_handoff_sent": {"bool": true},
+ });
+
+ let with_signals = State {
+ tool_signals: Some(ToolSignals::default()),
+ ..state.clone()
+ };
+ assert_eq!(serde_json::to_value(&with_signals)?, json);
+ assert_eq!(serde_json::from_value::<State>(json)?, state);
+ assert_eq!(with_signals, state);
+ Ok(())
+ }
}
diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rs
index d256a592e..74c8df8d9 100644
--- a/crates/libsy/src/core/testing.rs
+++ b/crates/libsy/src/core/testing.rs
@@ -24,7 +24,7 @@ use switchyard_protocol::{
};
use crate::core::algorithm::{Algorithm, Call, CallDecision, Driver, RuntimeModels};
-use crate::{LibsyError, Result};
+use crate::{LibsyError, Result, RoutingOutcome, State};
/// Builds one runtime model category for a test.
pub(crate) fn category_models(
@@ -94,6 +94,26 @@ pub(crate) async fn test_drive_with_models(
Ok((selected_model, response))
}
+/// Drive one request from caller `state` and return the routing outcome, which carries the
+/// updated state.
+pub(crate) async fn test_drive_with_state(
+ algorithm: Arc<dyn Algorithm>,
+ request: Request,
+ models: impl Into<RuntimeModels>,
+ state: State,
+ serve: impl Serve,
+) -> Result<RoutingOutcome> {
+ let serve = Arc::new(serve);
+ crate::core::algorithm::drive_with_state(
+ algorithm,
+ request,
+ Arc::new(models.into()),
+ state,
+ move |call| fulfill(Arc::clone(&serve), call),
+ )
+ .await
+}
+
pub(crate) async fn serve_decision(call: CallDecision) -> Result<()> {
let response = DecisionResponse {
id: None,
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 0c091db87..fe4d39fa3 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -45,9 +45,9 @@ pub use algorithms::vgr::{
// Stage-router scoring and tier selection — the shared signal-driven routing
// core (scorer, picker, and the `StageClassifier`).
pub use algorithms::util::stage::{
- CodingAgentDimensions, DECISION_SOURCE_KEY, DecisionSource, HandoffNoteConfig, PickOutcome,
- PickerMode, ScoreResult, StageClassifier, Tier, clear_fall_open, dimensions_from_signal,
- pick_tier, score_signal, set_fall_open,
+ CodingAgentDimensions, DecisionSource, HandoffNoteConfig, PickOutcome, PickerMode, ScoreResult,
+ StageClassifier, Tier, clear_fall_open, dimensions_from_signal, pick_tier, score_signal,
+ set_fall_open,
};
mod observability;
diff --git a/crates/prefill-router/src/algorithm.rs b/crates/prefill-router/src/algorithm.rs
index debe19f05..75908a778 100644
--- a/crates/prefill-router/src/algorithm.rs
+++ b/crates/prefill-router/src/algorithm.rs
@@ -9,6 +9,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use libsy::{
AffinityRouter, Algorithm, Classifier, Driver, Event, LibsyError, Processor, RoutingOutcome,
+ State,
};
use switchyard_protocol::{ContentBlock, Message, ModelId, Request, Role};
use tokio::sync::Mutex;
@@ -148,7 +149,7 @@ impl Algorithm for PrefillRouterAlgo {
driver: Driver,
mut request: Request,
) -> libsy::Result<RoutingOutcome> {
- let mut state = ();
+ let mut state = State::default();
if let Some(target) = self
.affinity
.score(&mut state, &mut request, &driver)
@@ -197,7 +198,7 @@ impl Algorithm for PrefillRouterAlgo {
self.default_target.clone()
}
};
- let mut state = ();
+ let mut state = State::default();
self.affinity
.process(
&mut state,
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index aa98c91db..83de01ff1 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -18,7 +18,7 @@ use switchyard_libsy::{
DecisionJudgeConfig, DeescalationConfig, EscalationJudgeConfig, HandoffNoteConfig,
LibsyError as RustLibsyError, LlmCapabilityConfig, LlmClassifierConfig, LlmFallback,
LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, StageRouter,
- StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics,
+ StageRouterConfig, State, Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics,
};
use switchyard_protocol::{
Category, DecisionResponse, LlmClientError, LlmResponse, LlmResponseStream,
@@ -911,7 +911,7 @@ impl PyAlgorithm {
}
let stream = {
let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();
- Arc::clone(&self.inner).run_stream(request, Arc::new(runtime_models))
+ Arc::clone(&self.inner).run_stream(request, Arc::new(runtime_models), State::default())
};
Ok(PyRunStream {
inner: Arc::new(Mutex::new(stream)),
@@ -941,6 +941,7 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {
request,
response,
metadata,
+ state: _,
} = *outcome;
Python::attach(|py| {
Ok(PyStep::Done {
diff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.md
index c13e7e47f..0e77945c0 100644
--- a/docs/routing_algorithms/composite_routing.md
+++ b/docs/routing_algorithms/composite_routing.md
@@ -124,6 +124,13 @@ conversation keeps its own. A client that sends no session ID can set
`classifier.message_hash_fallback = true`, which keys on the first user message
instead. Conversations that open with the same text then share a tier.
+Rust code that calls `Algorithm::run_stream` directly can store the tier
+itself. Pass the stored `State` to `run_stream`, save `RoutingOutcome.state`
+after each turn, and send no session ID. With a session ID, the router uses its
+own session state and returns the passed `State` unchanged. `switchyard-server`
+starts each request with an empty `State`, so server clients still send
+`x-switchyard-session-id`.
+
## Example use
An interactive coding agent in auto-mode. The classifier reads the user turn and