Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): preserve plan-execute sessions at capacity (#921)
ryan-lempka4d5ca0cf2fca
diff --git a/crates/libsy/src/algorithms/plan_execute.rs b/crates/libsy/src/algorithms/plan_execute.rsindex 74e4063a2..422003e0b 100644--- a/crates/libsy/src/algorithms/plan_execute.rs+++ b/crates/libsy/src/algorithms/plan_execute.rs@@ -80,40 +80,45 @@ impl PlanExecute {})}- fn phase(&self, request: &Request) -> Phase {+ fn phase(&self, request: &Request) -> Result<Phase> {let signals = ToolSignals::from_request(request, None);let mutation_seen = signals.edit_count > 0 || signals.write_count > 0;let Some(identity) = RoutingIdentity::from_request(request) else {- return if mutation_seen {+ return Ok(if mutation_seen {Phase::Handoff} else {Phase::Plan- };+ });};+ let is_session_final = request+ .metadata+ .as_ref()+ .and_then(|metadata| metadata.session_final)+ == Some(true);let mut sessions = self.executing_sessions.lock();let phase = if sessions.contains(&identity) {Phase::Execute} else if mutation_seen {- if sessions.len() >= MAX_EXECUTING_SESSIONS- && let Some(evicted) = sessions.iter().next().cloned()- {- sessions.remove(&evicted);+ if !is_session_final {+ if sessions.len() >= MAX_EXECUTING_SESSIONS {+ return Err(LibsyError::AlgorithmError {+ message: format!(+ "plan_execute reached its limit of {MAX_EXECUTING_SESSIONS} executing sessions. \+ Finish an existing session with session_final before retrying the handoff"+ ),+ });+ }+ sessions.insert(identity.clone());}- sessions.insert(identity.clone());Phase::Handoff} else {Phase::Plan};- if request- .metadata- .as_ref()- .and_then(|metadata| metadata.session_final)- == Some(true)- {+ if is_session_final {sessions.remove(&identity);}- phase+ Ok(phase)}fn replay_planner_reasoning_as_text(request: &mut Request) -> usize {@@ -166,7 +171,7 @@ impl Algorithm for PlanExecute {driver: Driver,mut request: Request,) -> Result<RoutingOutcome> {- match self.phase(&request) {+ match self.phase(&request)? {Phase::Plan => {prepend_system_prompt(&mut request, &self.config.planning_prompt);tracing::debug!(phase = "plan", "plan-execute selected capable tier");@@ -347,6 +352,78 @@ mod tests {assert_eq!(selected, CAPABLE);}+ #[tokio::test]+ async fn capacity_preserves_execution_after_compaction() {+ let algorithm = algorithm(PlanExecuteConfig::default());+ let sessions: Vec<_> = (0..MAX_EXECUTING_SESSIONS)+ .map(|index| format!("task-{index}"))+ .collect();+ for session in &sessions {+ let first_edit = request(+ vec![tool_call("Write", json!({"file_path": "task.py"}))],+ Some(session),+ );+ let (selected, _) = route_and_capture(Arc::clone(&algorithm), first_edit).await;+ assert_eq!(selected, EFFICIENT);+ }++ let overflow = request(+ vec![tool_call("Write", json!({"file_path": "task.py"}))],+ Some("overflow"),+ );+ let (driver, _) = Driver::new("plan_execute", Arc::new(models()));+ let result = Arc::clone(&algorithm).route(driver, overflow.clone()).await;+ assert!(matches!(+ result,+ Err(LibsyError::AlgorithmError { message })+ if message.contains("limit of 4096 executing sessions")+ ));++ for session in &sessions {+ let mut compacted = request(+ vec![Message::text(Role::User, "Continue after compaction")],+ Some(session),+ );+ if Some(session) == sessions.last() {+ compacted.metadata.as_mut().unwrap().session_final = Some(true);+ }+ let (selected, routed) = route_and_capture(Arc::clone(&algorithm), compacted).await;+ assert_eq!(selected, EFFICIENT, "session {session} lost its latch");+ assert!(routed.llm_request.instructions.is_empty());+ }++ let (selected, _) = route_and_capture(Arc::clone(&algorithm), overflow).await;+ assert_eq!(selected, EFFICIENT);+ let compacted = request(+ vec![Message::text(Role::User, "Continue after compaction")],+ Some("overflow"),+ );+ let (selected, routed) = route_and_capture(algorithm, compacted).await;+ assert_eq!(selected, EFFICIENT);+ assert!(routed.llm_request.instructions.is_empty());+ }++ #[tokio::test]+ async fn final_handoff_does_not_evict_an_executing_session_at_capacity() {+ let algorithm = Arc::new(PlanExecute::new(PlanExecuteConfig::default()).unwrap());+ algorithm.executing_sessions.lock().extend(+ (0..MAX_EXECUTING_SESSIONS)+ .map(|index| RoutingIdentity::Session(format!("task-{index}"))),+ );+ let mut final_request = request(+ vec![tool_call("Write", json!({"file_path": "task.py"}))],+ Some("final-handoff"),+ );+ final_request.metadata.as_mut().unwrap().session_final = Some(true);++ let (selected, routed) = route_and_capture(algorithm.clone(), final_request).await;+ assert_eq!(selected, EFFICIENT);+ assert!(routed.llm_request.instructions.is_empty());+ let sessions = algorithm.executing_sessions.lock();+ assert_eq!(sessions.len(), MAX_EXECUTING_SESSIONS);+ assert!(!sessions.contains(&RoutingIdentity::Session("final-handoff".to_string())));+ }+#[tokio::test]async fn mutation_without_a_session_uses_the_efficient_tier() {let messages = vec![tool_call(diff --git a/docs/routing_algorithms/plan_execute_routing.md b/docs/routing_algorithms/plan_execute_routing.mdindex 6efab06b1..a94bb46c4 100644--- a/docs/routing_algorithms/plan_execute_routing.md+++ b/docs/routing_algorithms/plan_execute_routing.md@@ -32,6 +32,22 @@ The first edit or write routes the full trajectory to the efficient target andlatches that choice by session ID. A failed edit still triggers the handoff.Without a session ID, the first mutation must remain in the request history.+## Session capacity++Each router instance retains up to 4,096 executing identities in memory. Root+requests use the session ID. Subagent requests use the session ID and agent ID.+Once retained, an identity stays on the efficient target after history compaction+until a request marks it with `session_final: true` or the router restarts.++At capacity, a new identity's handoff returns an error before calling a model.+Existing identities keep their execution phase. Mark an existing identity's last+request with `session_final: true` to free its slot, then retry the handoff with+the mutation history intact. The server accepts this flag through the+`x-switchyard-session-final: true` header.++A handoff marked final needs no saved slot and can still run at capacity.+Requests that are still planning or have no stable identity also use no slots.+## Responses API history requirementPlan/execute needs the conversation history to detect edits and hand the task to