Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(composite): retain profiles across compaction summaries (#1200)
ryan-lempkaf60a7a2291ea
diff --git a/Cargo.lock b/Cargo.lockindex 06b27d104..cdbf78a0c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2544,6 +2544,7 @@ dependencies = ["serde_json","shlex","switchyard-protocol",+ "switchyard-translation","thiserror 2.0.18","tokio","tokio-stream",@@ -2613,6 +2614,7 @@ dependencies = ["serde_json","switchyard-libsy","switchyard-protocol",+ "switchyard-translation",][[package]]@@ -2764,6 +2766,7 @@ dependencies = ["async-stream","base64","futures",+ "http","pretty_assertions","serde","serde_json",diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.tomlindex 950553af3..2e0a08169 100644--- a/crates/libsy/Cargo.toml+++ b/crates/libsy/Cargo.toml@@ -46,6 +46,7 @@ tokio.workspace = truetokio = { version = "1", features = ["rt", "sync", "macros", "time"] }[dev-dependencies]+switchyard-translation.workspace = true# SDK + in-memory exporter to assert what the observability layer records.opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }tokio.workspace = truediff --git a/crates/libsy/src/algorithms/composite.rs b/crates/libsy/src/algorithms/composite.rsindex a426c7f34..8e259bc14 100644--- a/crates/libsy/src/algorithms/composite.rs+++ b/crates/libsy/src/algorithms/composite.rs@@ -54,7 +54,10 @@ impl TierSetter {/// judge call sits between here and [`retain`](Self::retain). The later wins.fn is_due(&self, identity: Option<&RoutingIdentity>, state: &State, request: &Request) -> bool {match self.trigger {- ClassifyTrigger::UserTurn => has_new_user_turn(&request.llm_request.messages),+ ClassifyTrigger::UserTurn => {+ !request.llm_request.is_compaction+ && has_new_user_turn(&request.llm_request.messages)+ }// Unkeyed requests with no carried tier cannot be told apart, so every one// is a new session.ClassifyTrigger::NewSession => {@@ -332,6 +335,27 @@ mod tests {router_with_trigger(ClassifyTrigger::UserTurn)}+ #[tokio::test]+ async fn compaction_retains_the_profile_until_a_real_user_turn() -> Result<()> {+ let router = router()?;+ let recorder = Arc::new(Recorder::default());+ let mut compacted = user_turn_request();+ compacted.llm_request.is_compaction = true;+ for (request, verdict, calls, target) in [+ (user_turn_request(), 1.0, 1, "weak"),+ (compacted, 0.0, 1, "weak"),+ (user_turn_request(), 0.0, 2, "strong"),+ ] {+ *recorder.judge_p_solve.lock() = verdict;+ let (model, _) =+ test_drive_with_models(router.clone(), request, runtime_models(), recorder.serve())+ .await?;+ assert_eq!(recorder.judge_calls(), calls);+ assert_eq!(model.as_str(), target);+ }+ Ok(())+ }+#[tokio::test]async fn capable_first_defaults_to_pinning_efficient_after_a_switch() -> Result<()> {for capable_first in [false, true] {diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rsindex a5eb17519..70040f9bd 100644--- a/crates/libsy/src/algorithms/util/tool_signals.rs+++ b/crates/libsy/src/algorithms/util/tool_signals.rs@@ -1068,7 +1068,6 @@ fn extract_tool_signals_with_window_and_semantics(let mut tool_calls: Vec<ObservedToolCall> = Vec::new();// IDs whose latest call is a retrieval tool.let mut retrieval_calls: HashSet<&str> = HashSet::new();- let mut compacted = false;let mut tool_result_count = 0usize;let mut assistant_turn_count = 0usize;@@ -1078,8 +1077,7 @@ fn extract_tool_signals_with_window_and_semantics(}for block in &message.content {match block {- // Only context-compaction markers carry across user turns.- _ if index < start && !matches!(block, ContentBlock::Text { .. }) => continue,+ _ if index < start => continue,ContentBlock::ToolCall(call) => {// Responses namespaced tools arrive as `<namespace>__<tool>`.let bare_name = namespaces@@ -1245,18 +1243,6 @@ fn extract_tool_signals_with_window_and_semantics(let text = texts.join("\n");tool_texts.push((text, is_error));}- // Compaction is detected anywhere in the conversation: the summary- // stays in the prefix on every later turn, so this self-latches- // once it fires.- ContentBlock::Text { text } => {- let text = text.to_lowercase();- compacted |= text.contains(COMPACTION_MARKER)- || text.lines().any(|line| {- HERMES_COMPACTION_MARKERS- .iter()- .any(|marker| line.trim_start().starts_with(marker))- });- }_ => {}}}@@ -1269,22 +1255,12 @@ fn extract_tool_signals_with_window_and_semantics(recent_window,semantics,);- signal.compacted = compacted;+ signal.compacted = request.llm_request.has_compaction;signal.tool_result_count = u32::try_from(tool_result_count).unwrap_or(u32::MAX);signal.assistant_turn_count = u32::try_from(assistant_turn_count).unwrap_or(u32::MAX);signal}-/// Distinctive preamble Claude Code injects as a user message when it compacts an-/// overflowed context. Matched case-insensitively; normal task text never contains it.-const COMPACTION_MARKER: &str = "session is being continued";--// Hermes can merge the summary into an existing message or restate an active task.-const HERMES_COMPACTION_MARKERS: &[&str] = &[- "[context compaction \u{2014} reference only]",- "[still in progress \u{2014} this is the active request, restated after the compaction boundary",-];-/// The tool part of an `mcp__<server>__<tool>` name, the form Claude Code uses/// for MCP tools. The server name is assumed not to contain `__`; the tool name/// may.@@ -1761,14 +1737,16 @@ mod tests {};fn with_messages(messages: Vec<Message>) -> Request {- Request {+ let mut request = Request {llm_request: LlmRequest {messages,..LlmRequest::default()},raw_request: None,metadata: None,- }+ };+ switchyard_translation::normalize_compaction(&mut request.llm_request, None);+ request}// assistant message with a single named tool calldiff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rsindex cb3731aef..122412615 100644--- a/crates/protocol/src/llm.rs+++ b/crates/protocol/src/llm.rs@@ -536,6 +536,12 @@ pub struct LlmRequest {pub instructions: Vec<InstructionBlock>,/// Ordered conversation messages.pub messages: Vec<Message>,+ /// Whether this request performs compaction or resumes from a summary.+ #[serde(skip_serializing_if = "std::ops::Not::not")]+ pub is_compaction: bool,+ /// Whether the history contains a recognized compaction marker.+ #[serde(skip_serializing_if = "std::ops::Not::not")]+ pub has_compaction: bool,/// Tools available to the model.pub tools: Vec<Tool>,/// Policy controlling model tool selection.diff --git a/crates/switchyard-ffi/src/lib.rs b/crates/switchyard-ffi/src/lib.rsindex 30eedf79f..180d203e5 100644--- a/crates/switchyard-ffi/src/lib.rs+++ b/crates/switchyard-ffi/src/lib.rs@@ -551,7 +551,7 @@ pub unsafe extern "C" fn sy_run_new(if let Some(subagent_models) = input.subagent_models {models = models.with_subagent(parse_model_groups(subagent_models)?);}- let request = Request {+ let mut request = Request {llm_request: input.request,metadata: input.metadata.map(|metadata| Metadata {session_id: metadata.session_id,@@ -563,6 +563,7 @@ pub unsafe extern "C" fn sy_run_new(}),..Default::default()};+ switchyard_translation::normalize_compaction(&mut request.llm_request, None);let _entered = router.engine.runtime.enter();let stream = Arc::clone(&router.engine.algorithm).run_stream(request,diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rsindex e561ff8c9..d708ef7b5 100644--- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs+++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs@@ -119,6 +119,7 @@ impl SwitchyardRuntime {.map_err(|error| self.redactor.text(error))?;llm_request.stream = streaming;let headers = string_headers(&request.headers);+ switchyard_translation::normalize_compaction(&mut llm_request, Some(&headers));let mut metadata = Metadata::from_headers(&headers);let relay_gateway_placeholder = !headers.contains_key("x-switchyard-session-id")&& headersdiff --git a/crates/switchyard-node/Cargo.toml b/crates/switchyard-node/Cargo.tomlindex e008c132f..b35608ebf 100644--- a/crates/switchyard-node/Cargo.toml+++ b/crates/switchyard-node/Cargo.toml@@ -24,6 +24,7 @@ serde.workspace = trueserde_json.workspace = trueswitchyard-libsy.workspace = trueswitchyard-protocol.workspace = true+switchyard-translation.workspace = true[build-dependencies]napi-build = "2"diff --git a/crates/switchyard-node/src/lib.rs b/crates/switchyard-node/src/lib.rsindex d163a8817..72d28ad17 100644--- a/crates/switchyard-node/src/lib.rs+++ b/crates/switchyard-node/src/lib.rs@@ -158,7 +158,7 @@ impl NativeComposite {messages_json: String,state_json: Option<String>,) -> napi::Result<NativeRun> {- let request = Request {+ let mut request = Request {llm_request: LlmRequest {model: Some("switchyard/auto".into()),messages: parse(&messages_json)?,@@ -166,6 +166,7 @@ impl NativeComposite {},..Default::default()};+ switchyard_translation::normalize_compaction(&mut request.llm_request, None);let initial_state: State = state_json.as_deref().map(parse)diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 3635ed50b..131208795 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -84,8 +84,10 @@ fn request_from_python(headers: Option<HashMap<String, String>>,) -> PyResult<Request> {let headers = headers.as_ref().map(header_map_from_python).transpose()?;+ let mut llm_request = from_python(request)?;+ switchyard_translation::normalize_compaction(&mut llm_request, headers.as_ref());Ok(Request {- llm_request: from_python(request)?,+ llm_request,raw_request: None,metadata: headers.map(|headers| Metadata::from_headers(&headers)),})diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex d1bb3c4cc..2c0a87c91 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -1023,8 +1023,9 @@ fn resolve_route(if let Some(metadata) = body.get_mut("metadata").and_then(Value::as_object_mut) {metadata.remove(switchyard_translation::util::SWITCHYARD_METADATA_KEY);}- let llm_request = observed_translation::decode_request(wire_format, &body)+ let mut llm_request = observed_translation::decode_request(wire_format, &body).map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?;+ switchyard_translation::normalize_compaction(&mut llm_request, metadata.http_headers.as_ref());let requested_model = llm_request.model.clone()diff --git a/crates/switchyard-translation/Cargo.toml b/crates/switchyard-translation/Cargo.tomlindex bb121d931..d765ca205 100644--- a/crates/switchyard-translation/Cargo.toml+++ b/crates/switchyard-translation/Cargo.toml@@ -17,6 +17,7 @@ keywords = ["llm", "translation", "openai", "anthropic"]publish = ["crates-io"][dependencies]+http.workspace = truebase64.workspace = trueserde.workspace = trueserde_json.workspace = truediff --git a/crates/switchyard-translation/src/compaction.rs b/crates/switchyard-translation/src/compaction.rsnew file mode 100644index 000000000..cc51b64ed--- /dev/null+++ b/crates/switchyard-translation/src/compaction.rs@@ -0,0 +1,125 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use http::HeaderMap;+use switchyard_protocol::{ContentBlock, LlmRequest, Message, Role};++/// Normalizes current and historical compaction signals, preserving explicit IR annotations.+pub fn normalize_compaction(request: &mut LlmRequest, headers: Option<&HeaderMap>) {+ let is_compaction = headers+ .and_then(|headers| headers.get("x-codex-turn-metadata"))+ .and_then(|value| serde_json::from_slice::<serde_json::Value>(value.as_bytes()).ok())+ .is_some_and(|metadata| metadata["request_kind"] == "compaction");+ request.has_compaction |= request+ .messages+ .iter()+ .flat_map(|message| &message.content)+ .any(|block| match block {+ ContentBlock::Text { text } => {+ let text = text.to_lowercase();+ text.contains("session is being continued") || has_hermes_marker(&text)+ }+ _ => false,+ });+ request.is_compaction |= is_compaction+ || request+ .messages+ .iter()+ .rev()+ .find(|message| !is_environment_update(message))+ .is_some_and(|message| message.role == Role::User && is_compaction_message(message));+}++fn is_environment_update(message: &Message) -> bool {+ message.role == Role::User+ && !message.content.is_empty()+ && message.content.iter().all(|block| {+ matches!(block, ContentBlock::Text { text } if text.trim()+ .strip_prefix("<environment_context>")+ .and_then(|context| context.split_once("</environment_context>"))+ .is_some_and(|(_, rest)| rest.trim().is_empty()))+ })+}++fn is_compaction_message(message: &Message) -> bool {+ let mut has_summary = false;+ for block in &message.content {+ match block {+ ContentBlock::Image { .. } => {}+ _ if is_compaction_summary(block) => has_summary = true,+ _ => return false,+ }+ }+ has_summary+}++fn is_compaction_summary(block: &ContentBlock) -> bool {+ match block {+ ContentBlock::Unknown { raw, .. } => matches!(+ raw.get("type").and_then(serde_json::Value::as_str),+ Some("compaction" | "context_compaction")+ ),+ ContentBlock::Text { text } => {+ let text = text.trim().to_ascii_lowercase();+ text.starts_with("another language model started to solve this problem and produced a summary of its thinking process.")+ || text.starts_with("this session is being continued from a previous conversation that ran out of context.")+ || (text.starts_with("the conversation history before this point was compacted into the following summary:")+ && text.ends_with("</summary>"))+ || (text.starts_with("prior model work/tool state available.")+ && text.ends_with("</summary>"))+ || (text.starts_with("<conversation-checkpoint>")+ && text.ends_with("</conversation-checkpoint>"))+ || is_hermes_summary(&text)+ }+ _ => false,+ }+}++const HERMES_REPLAY: &str = "[still in progress \u{2014} this is the active request, restated after the compaction boundary";++fn is_hermes_summary(text: &str) -> bool {+ const END: &str = "--- end of context summary \u{2014} respond to the message below, not the summary above ---";+ if let Some((_, tail)) = text.rsplit_once(END)+ && !tail.trim().is_empty()+ && !tail.trim_start().starts_with(HERMES_REPLAY)+ {+ return false;+ }+ has_hermes_marker(text)+ || text+ .lines()+ .any(|line| line.trim_start().starts_with("[context summary]:"))+}++fn has_hermes_marker(text: &str) -> bool {+ text.lines().any(|line| {+ let line = line.trim_start();+ line.starts_with("[context compaction \u{2014} reference only]")+ || line.starts_with(HERMES_REPLAY)+ })+}++#[cfg(test)]+mod tests {+ use crate::{WireFormat, decode_request};+ use serde_json::json;++ #[test]+ fn summary_continues_but_fresh_user_input_does_not() {+ let mut body = json!({"messages": [+ {"role": "user", "content": [+ {"type": "text", "text": "prior context\n[CONTEXT COMPACTION \u{2014} REFERENCE ONLY] summary"},+ {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}+ ]},+ {"role": "user", "content": "<environment_context>updated</environment_context>"}+ ]});+ let request = decode_request(WireFormat::OpenAiChat, &body).unwrap();+ assert!(request.is_compaction && request.has_compaction);+ body["messages"]+ .as_array_mut()+ .unwrap()+ .push(json!({"role": "user", "content": "Now fix the parser"}));+ let followup = decode_request(WireFormat::OpenAiChat, &body).unwrap();+ assert!(!followup.is_compaction && followup.has_compaction);+ }+}diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rsindex 3399c769e..ae2020a87 100644--- a/crates/switchyard-translation/src/engine.rs+++ b/crates/switchyard-translation/src/engine.rs@@ -119,7 +119,8 @@ impl TranslationEngine {policy: &TranslationPolicy,) -> Result<RequestIrOutput> {let source = source.into();- let decoded = self.registry.codec(source)?.decode_request(body, policy)?;+ let mut decoded = self.registry.codec(source)?.decode_request(body, policy)?;+ crate::normalize_compaction(&mut decoded.request, None);Ok(RequestIrOutput {request: decoded.request,diagnostics: decoded.diagnostics,diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rsindex 568959097..3b23417cc 100644--- a/crates/switchyard-translation/src/lib.rs+++ b/crates/switchyard-translation/src/lib.rs@@ -6,6 +6,7 @@pub mod codecs;pub(crate) mod codex_custom_tools;pub(crate) mod codex_namespaces;+mod compaction;pub mod diagnostic;pub mod engine;pub mod error;@@ -22,6 +23,7 @@ pub use switchyard_protocol::stream::{};pub use switchyard_protocol::{format, llm};+pub use compaction::normalize_compaction;pub use diagnostic::*;pub use engine::*;pub use error::*;diff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.mdindex 3f524cacf..596cb7886 100644--- a/docs/routing_algorithms/composite_routing.md+++ b/docs/routing_algorithms/composite_routing.md@@ -58,6 +58,9 @@ new = ["send_message"]`[routes.switchyard.classifier]` is required and takes the `stage_router` classifier fields.`classify_trigger` sets how often the judge runs: `user_turn` re-picks the tierwhenever the user speaks, `new_session` picks once and holds it.+Compaction continuations keep the selected profile. Translation recognizes native+signals and known summary wrappers, then sets `is_compaction` in the IR. Stage reads+`has_compaction` for its existing history check. Direct IR callers can set these flags.`[routes.switchyard.stage]` takes the stage settings, but accepts neither `picker`nor a second `classifier`. The outer classifier supplies the default tier.`confidence_threshold` remains an alias for `efficient_first_threshold`.