Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(composite): retain profiles across compaction summaries (#1200)

ryan-lempkaf60a7a2291ea
diff --git a/Cargo.lock b/Cargo.lock
index 06b27d104..cdbf78a0c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2544,6 +2544,7 @@ dependencies = [
"serde_json",
"shlex",
"switchyard-protocol",
+ "switchyard-translation",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
@@ -2613,6 +2614,7 @@ dependencies = [
"serde_json",
"switchyard-libsy",
"switchyard-protocol",
+ "switchyard-translation",
]
[[package]]
@@ -2764,6 +2766,7 @@ dependencies = [
"async-stream",
"base64",
"futures",
+ "http",
"pretty_assertions",
"serde",
"serde_json",
diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml
index 950553af3..2e0a08169 100644
--- a/crates/libsy/Cargo.toml
+++ b/crates/libsy/Cargo.toml
@@ -46,6 +46,7 @@ tokio.workspace = true
tokio = { version = "1", features = ["rt", "sync", "macros", "time"] }
[dev-dependencies]
+switchyard-translation.workspace = true
# SDK + in-memory exporter to assert what the observability layer records.
opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }
tokio.workspace = true
diff --git a/crates/libsy/src/algorithms/composite.rs b/crates/libsy/src/algorithms/composite.rs
index a426c7f34..8e259bc14 100644
--- a/crates/libsy/src/algorithms/composite.rs
+++ b/crates/libsy/src/algorithms/composite.rs
@@ -54,7 +54,10 @@ impl TierSetter {
/// judge call sits between here and [`retain`](Self::retain). The later wins.
fn is_due(&self, identity: Option<&RoutingIdentity>, state: &State, request: &Request) -> bool {
match self.trigger {
- ClassifyTrigger::UserTurn => has_new_user_turn(&request.llm_request.messages),
+ ClassifyTrigger::UserTurn => {
+ !request.llm_request.is_compaction
+ && has_new_user_turn(&request.llm_request.messages)
+ }
// Unkeyed requests with no carried tier cannot be told apart, so every one
// is a new session.
ClassifyTrigger::NewSession => {
@@ -332,6 +335,27 @@ mod tests {
router_with_trigger(ClassifyTrigger::UserTurn)
}
+ #[tokio::test]
+ async fn compaction_retains_the_profile_until_a_real_user_turn() -> Result<()> {
+ let router = router()?;
+ let recorder = Arc::new(Recorder::default());
+ let mut compacted = user_turn_request();
+ compacted.llm_request.is_compaction = true;
+ for (request, verdict, calls, target) in [
+ (user_turn_request(), 1.0, 1, "weak"),
+ (compacted, 0.0, 1, "weak"),
+ (user_turn_request(), 0.0, 2, "strong"),
+ ] {
+ *recorder.judge_p_solve.lock() = verdict;
+ let (model, _) =
+ test_drive_with_models(router.clone(), request, runtime_models(), recorder.serve())
+ .await?;
+ assert_eq!(recorder.judge_calls(), calls);
+ assert_eq!(model.as_str(), target);
+ }
+ Ok(())
+ }
+
#[tokio::test]
async fn capable_first_defaults_to_pinning_efficient_after_a_switch() -> Result<()> {
for capable_first in [false, true] {
diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs
index a5eb17519..70040f9bd 100644
--- a/crates/libsy/src/algorithms/util/tool_signals.rs
+++ b/crates/libsy/src/algorithms/util/tool_signals.rs
@@ -1068,7 +1068,6 @@ fn extract_tool_signals_with_window_and_semantics(
let mut tool_calls: Vec<ObservedToolCall> = Vec::new();
// IDs whose latest call is a retrieval tool.
let mut retrieval_calls: HashSet<&str> = HashSet::new();
- let mut compacted = false;
let mut tool_result_count = 0usize;
let mut assistant_turn_count = 0usize;
@@ -1078,8 +1077,7 @@ fn extract_tool_signals_with_window_and_semantics(
}
for block in &message.content {
match block {
- // Only context-compaction markers carry across user turns.
- _ if index < start && !matches!(block, ContentBlock::Text { .. }) => continue,
+ _ if index < start => continue,
ContentBlock::ToolCall(call) => {
// Responses namespaced tools arrive as `<namespace>__<tool>`.
let bare_name = namespaces
@@ -1245,18 +1243,6 @@ fn extract_tool_signals_with_window_and_semantics(
let text = texts.join("\n");
tool_texts.push((text, is_error));
}
- // Compaction is detected anywhere in the conversation: the summary
- // stays in the prefix on every later turn, so this self-latches
- // once it fires.
- ContentBlock::Text { text } => {
- let text = text.to_lowercase();
- compacted |= text.contains(COMPACTION_MARKER)
- || text.lines().any(|line| {
- HERMES_COMPACTION_MARKERS
- .iter()
- .any(|marker| line.trim_start().starts_with(marker))
- });
- }
_ => {}
}
}
@@ -1269,22 +1255,12 @@ fn extract_tool_signals_with_window_and_semantics(
recent_window,
semantics,
);
- signal.compacted = compacted;
+ signal.compacted = request.llm_request.has_compaction;
signal.tool_result_count = u32::try_from(tool_result_count).unwrap_or(u32::MAX);
signal.assistant_turn_count = u32::try_from(assistant_turn_count).unwrap_or(u32::MAX);
signal
}
-/// Distinctive preamble Claude Code injects as a user message when it compacts an
-/// overflowed context. Matched case-insensitively; normal task text never contains it.
-const COMPACTION_MARKER: &str = "session is being continued";
-
-// Hermes can merge the summary into an existing message or restate an active task.
-const HERMES_COMPACTION_MARKERS: &[&str] = &[
- "[context compaction \u{2014} reference only]",
- "[still in progress \u{2014} this is the active request, restated after the compaction boundary",
-];
-
/// The tool part of an `mcp__<server>__<tool>` name, the form Claude Code uses
/// for MCP tools. The server name is assumed not to contain `__`; the tool name
/// may.
@@ -1761,14 +1737,16 @@ mod tests {
};
fn with_messages(messages: Vec<Message>) -> Request {
- Request {
+ let mut request = Request {
llm_request: LlmRequest {
messages,
..LlmRequest::default()
},
raw_request: None,
metadata: None,
- }
+ };
+ switchyard_translation::normalize_compaction(&mut request.llm_request, None);
+ request
}
// assistant message with a single named tool call
diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs
index cb3731aef..122412615 100644
--- a/crates/protocol/src/llm.rs
+++ b/crates/protocol/src/llm.rs
@@ -536,6 +536,12 @@ pub struct LlmRequest {
pub instructions: Vec<InstructionBlock>,
/// Ordered conversation messages.
pub messages: Vec<Message>,
+ /// Whether this request performs compaction or resumes from a summary.
+ #[serde(skip_serializing_if = "std::ops::Not::not")]
+ pub is_compaction: bool,
+ /// Whether the history contains a recognized compaction marker.
+ #[serde(skip_serializing_if = "std::ops::Not::not")]
+ pub has_compaction: bool,
/// Tools available to the model.
pub tools: Vec<Tool>,
/// Policy controlling model tool selection.
diff --git a/crates/switchyard-ffi/src/lib.rs b/crates/switchyard-ffi/src/lib.rs
index 30eedf79f..180d203e5 100644
--- a/crates/switchyard-ffi/src/lib.rs
+++ b/crates/switchyard-ffi/src/lib.rs
@@ -551,7 +551,7 @@ pub unsafe extern "C" fn sy_run_new(
if let Some(subagent_models) = input.subagent_models {
models = models.with_subagent(parse_model_groups(subagent_models)?);
}
- let request = Request {
+ let mut request = Request {
llm_request: input.request,
metadata: input.metadata.map(|metadata| Metadata {
session_id: metadata.session_id,
@@ -563,6 +563,7 @@ pub unsafe extern "C" fn sy_run_new(
}),
..Default::default()
};
+ switchyard_translation::normalize_compaction(&mut request.llm_request, None);
let _entered = router.engine.runtime.enter();
let stream = Arc::clone(&router.engine.algorithm).run_stream(
request,
diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs
index e561ff8c9..d708ef7b5 100644
--- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs
+++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs
@@ -119,6 +119,7 @@ impl SwitchyardRuntime {
.map_err(|error| self.redactor.text(error))?;
llm_request.stream = streaming;
let headers = string_headers(&request.headers);
+ switchyard_translation::normalize_compaction(&mut llm_request, Some(&headers));
let mut metadata = Metadata::from_headers(&headers);
let relay_gateway_placeholder = !headers.contains_key("x-switchyard-session-id")
&& headers
diff --git a/crates/switchyard-node/Cargo.toml b/crates/switchyard-node/Cargo.toml
index e008c132f..b35608ebf 100644
--- a/crates/switchyard-node/Cargo.toml
+++ b/crates/switchyard-node/Cargo.toml
@@ -24,6 +24,7 @@ serde.workspace = true
serde_json.workspace = true
switchyard-libsy.workspace = true
switchyard-protocol.workspace = true
+switchyard-translation.workspace = true
[build-dependencies]
napi-build = "2"
diff --git a/crates/switchyard-node/src/lib.rs b/crates/switchyard-node/src/lib.rs
index d163a8817..72d28ad17 100644
--- a/crates/switchyard-node/src/lib.rs
+++ b/crates/switchyard-node/src/lib.rs
@@ -158,7 +158,7 @@ impl NativeComposite {
messages_json: String,
state_json: Option<String>,
) -> napi::Result<NativeRun> {
- let request = Request {
+ let mut request = Request {
llm_request: LlmRequest {
model: Some("switchyard/auto".into()),
messages: parse(&messages_json)?,
@@ -166,6 +166,7 @@ impl NativeComposite {
},
..Default::default()
};
+ switchyard_translation::normalize_compaction(&mut request.llm_request, None);
let initial_state: State = state_json
.as_deref()
.map(parse)
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 3635ed50b..131208795 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -84,8 +84,10 @@ fn request_from_python(
headers: Option<HashMap<String, String>>,
) -> PyResult<Request> {
let headers = headers.as_ref().map(header_map_from_python).transpose()?;
+ let mut llm_request = from_python(request)?;
+ switchyard_translation::normalize_compaction(&mut llm_request, headers.as_ref());
Ok(Request {
- llm_request: from_python(request)?,
+ llm_request,
raw_request: None,
metadata: headers.map(|headers| Metadata::from_headers(&headers)),
})
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index d1bb3c4cc..2c0a87c91 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -1023,8 +1023,9 @@ fn resolve_route(
if let Some(metadata) = body.get_mut("metadata").and_then(Value::as_object_mut) {
metadata.remove(switchyard_translation::util::SWITCHYARD_METADATA_KEY);
}
- let llm_request = observed_translation::decode_request(wire_format, &body)
+ let mut llm_request = observed_translation::decode_request(wire_format, &body)
.map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?;
+ switchyard_translation::normalize_compaction(&mut llm_request, metadata.http_headers.as_ref());
let requested_model = llm_request
.model
.clone()
diff --git a/crates/switchyard-translation/Cargo.toml b/crates/switchyard-translation/Cargo.toml
index bb121d931..d765ca205 100644
--- a/crates/switchyard-translation/Cargo.toml
+++ b/crates/switchyard-translation/Cargo.toml
@@ -17,6 +17,7 @@ keywords = ["llm", "translation", "openai", "anthropic"]
publish = ["crates-io"]
[dependencies]
+http.workspace = true
base64.workspace = true
serde.workspace = true
serde_json.workspace = true
diff --git a/crates/switchyard-translation/src/compaction.rs b/crates/switchyard-translation/src/compaction.rs
new file mode 100644
index 000000000..cc51b64ed
--- /dev/null
+++ b/crates/switchyard-translation/src/compaction.rs
@@ -0,0 +1,125 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use http::HeaderMap;
+use switchyard_protocol::{ContentBlock, LlmRequest, Message, Role};
+
+/// Normalizes current and historical compaction signals, preserving explicit IR annotations.
+pub fn normalize_compaction(request: &mut LlmRequest, headers: Option<&HeaderMap>) {
+ let is_compaction = headers
+ .and_then(|headers| headers.get("x-codex-turn-metadata"))
+ .and_then(|value| serde_json::from_slice::<serde_json::Value>(value.as_bytes()).ok())
+ .is_some_and(|metadata| metadata["request_kind"] == "compaction");
+ request.has_compaction |= request
+ .messages
+ .iter()
+ .flat_map(|message| &message.content)
+ .any(|block| match block {
+ ContentBlock::Text { text } => {
+ let text = text.to_lowercase();
+ text.contains("session is being continued") || has_hermes_marker(&text)
+ }
+ _ => false,
+ });
+ request.is_compaction |= is_compaction
+ || request
+ .messages
+ .iter()
+ .rev()
+ .find(|message| !is_environment_update(message))
+ .is_some_and(|message| message.role == Role::User && is_compaction_message(message));
+}
+
+fn is_environment_update(message: &Message) -> bool {
+ message.role == Role::User
+ && !message.content.is_empty()
+ && message.content.iter().all(|block| {
+ matches!(block, ContentBlock::Text { text } if text.trim()
+ .strip_prefix("<environment_context>")
+ .and_then(|context| context.split_once("</environment_context>"))
+ .is_some_and(|(_, rest)| rest.trim().is_empty()))
+ })
+}
+
+fn is_compaction_message(message: &Message) -> bool {
+ let mut has_summary = false;
+ for block in &message.content {
+ match block {
+ ContentBlock::Image { .. } => {}
+ _ if is_compaction_summary(block) => has_summary = true,
+ _ => return false,
+ }
+ }
+ has_summary
+}
+
+fn is_compaction_summary(block: &ContentBlock) -> bool {
+ match block {
+ ContentBlock::Unknown { raw, .. } => matches!(
+ raw.get("type").and_then(serde_json::Value::as_str),
+ Some("compaction" | "context_compaction")
+ ),
+ ContentBlock::Text { text } => {
+ let text = text.trim().to_ascii_lowercase();
+ text.starts_with("another language model started to solve this problem and produced a summary of its thinking process.")
+ || text.starts_with("this session is being continued from a previous conversation that ran out of context.")
+ || (text.starts_with("the conversation history before this point was compacted into the following summary:")
+ && text.ends_with("</summary>"))
+ || (text.starts_with("prior model work/tool state available.")
+ && text.ends_with("</summary>"))
+ || (text.starts_with("<conversation-checkpoint>")
+ && text.ends_with("</conversation-checkpoint>"))
+ || is_hermes_summary(&text)
+ }
+ _ => false,
+ }
+}
+
+const HERMES_REPLAY: &str = "[still in progress \u{2014} this is the active request, restated after the compaction boundary";
+
+fn is_hermes_summary(text: &str) -> bool {
+ const END: &str = "--- end of context summary \u{2014} respond to the message below, not the summary above ---";
+ if let Some((_, tail)) = text.rsplit_once(END)
+ && !tail.trim().is_empty()
+ && !tail.trim_start().starts_with(HERMES_REPLAY)
+ {
+ return false;
+ }
+ has_hermes_marker(text)
+ || text
+ .lines()
+ .any(|line| line.trim_start().starts_with("[context summary]:"))
+}
+
+fn has_hermes_marker(text: &str) -> bool {
+ text.lines().any(|line| {
+ let line = line.trim_start();
+ line.starts_with("[context compaction \u{2014} reference only]")
+ || line.starts_with(HERMES_REPLAY)
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::{WireFormat, decode_request};
+ use serde_json::json;
+
+ #[test]
+ fn summary_continues_but_fresh_user_input_does_not() {
+ let mut body = json!({"messages": [
+ {"role": "user", "content": [
+ {"type": "text", "text": "prior context\n[CONTEXT COMPACTION \u{2014} REFERENCE ONLY] summary"},
+ {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}
+ ]},
+ {"role": "user", "content": "<environment_context>updated</environment_context>"}
+ ]});
+ let request = decode_request(WireFormat::OpenAiChat, &body).unwrap();
+ assert!(request.is_compaction && request.has_compaction);
+ body["messages"]
+ .as_array_mut()
+ .unwrap()
+ .push(json!({"role": "user", "content": "Now fix the parser"}));
+ let followup = decode_request(WireFormat::OpenAiChat, &body).unwrap();
+ assert!(!followup.is_compaction && followup.has_compaction);
+ }
+}
diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs
index 3399c769e..ae2020a87 100644
--- a/crates/switchyard-translation/src/engine.rs
+++ b/crates/switchyard-translation/src/engine.rs
@@ -119,7 +119,8 @@ impl TranslationEngine {
policy: &TranslationPolicy,
) -> Result<RequestIrOutput> {
let source = source.into();
- let decoded = self.registry.codec(source)?.decode_request(body, policy)?;
+ let mut decoded = self.registry.codec(source)?.decode_request(body, policy)?;
+ crate::normalize_compaction(&mut decoded.request, None);
Ok(RequestIrOutput {
request: decoded.request,
diagnostics: decoded.diagnostics,
diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs
index 568959097..3b23417cc 100644
--- a/crates/switchyard-translation/src/lib.rs
+++ b/crates/switchyard-translation/src/lib.rs
@@ -6,6 +6,7 @@
pub mod codecs;
pub(crate) mod codex_custom_tools;
pub(crate) mod codex_namespaces;
+mod compaction;
pub mod diagnostic;
pub mod engine;
pub mod error;
@@ -22,6 +23,7 @@ pub use switchyard_protocol::stream::{
};
pub use switchyard_protocol::{format, llm};
+pub use compaction::normalize_compaction;
pub use diagnostic::*;
pub use engine::*;
pub use error::*;
diff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.md
index 3f524cacf..596cb7886 100644
--- a/docs/routing_algorithms/composite_routing.md
+++ b/docs/routing_algorithms/composite_routing.md
@@ -58,6 +58,9 @@ new = ["send_message"]
`[routes.switchyard.classifier]` is required and takes the `stage_router` classifier fields.
`classify_trigger` sets how often the judge runs: `user_turn` re-picks the tier
whenever the user speaks, `new_session` picks once and holds it.
+Compaction continuations keep the selected profile. Translation recognizes native
+signals and known summary wrappers, then sets `is_compaction` in the IR. Stage reads
+`has_compaction` for its existing history check. Direct IR callers can set these flags.
`[routes.switchyard.stage]` takes the stage settings, but accepts neither `picker`
nor a second `classifier`. The outer classifier supplies the default tier.
`confidence_threshold` remains an alias for `efficient_first_threshold`.