Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(client): materialize cross-format Responses state (#781)

sabhatinasee3715d10ad3
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 10b098f96..01e1c36fc 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -13,10 +13,9 @@
//! completion candidate exhausts its backend retry budget before fallback advances. Routing
//! calls stop on the first candidate's failure. A timeout stops either kind of call.
//!
-//! A Responses continuation can refer to state held by one provider through
-//! `previous_response_id` or `conversation`. When completion targets use different clients,
-//! [`ClientRouter`] records which model served that state and sends known continuations
-//! back to it without running the algorithm or trying another provider.
+//! A Responses continuation can refer to state through `previous_response_id` or `conversation`.
+//! [`ClientRouter`] sends native Responses state back to its provider; for Chat or Anthropic
+//! targets it retains the canonical history needed to materialize the continuation.
use std::collections::HashMap;
use std::sync::Arc;
@@ -30,8 +29,9 @@ use switchyard_libsy::{
Algorithm, CallModel, LibsyError, OutcomeMetadata, Result, RoutingOutcome, RuntimeModels, drive,
};
use switchyard_protocol::{
- LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, ModelId, Request, Response,
- RoutedLlmClient, RoutingFallbackReason, WireFormat,
+ AggLlmResponse, LlmClientError, LlmResponse, LlmResponseChunk, LlmResponseStream, Message,
+ ModelId, Request, Response, ResponseAccumulator, RoutedLlmClient, RoutingFallbackReason,
+ WireFormat,
};
use switchyard_translation::prepare_request_for_target;
@@ -392,12 +392,18 @@ fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {
}
/// Route to `owner` without fallbacks and record why the algorithm did not run.
-fn continue_on(owner: StateOwner, algorithm: &str, request: Request) -> RoutingOutcome {
+fn continue_on(owner: StateOwner, algorithm: &str, mut request: Request) -> RoutingOutcome {
tracing::debug!(
target = %owner.model,
trigger = owner.trigger,
"routing Responses continuation to the model that holds its state"
);
+ if let Some(history) = owner.history {
+ let mut messages = Vec::with_capacity(history.len + request.llm_request.messages.len());
+ history.extend(&mut messages);
+ messages.append(&mut request.llm_request.messages);
+ request.llm_request.messages = messages;
+ }
let mut outcome = RoutingOutcome::route_to(owner.model, Vec::new(), request);
outcome.metadata = Some(OutcomeMetadata::new(
algorithm.to_string(),
@@ -410,21 +416,64 @@ fn continue_on(owner: StateOwner, algorithm: &str, request: Request) -> RoutingO
struct StateOwner {
model: ModelId,
trigger: &'static str,
+ history: Option<Arc<MessageHistory>>,
+}
+
+struct MessageHistory {
+ parent: Option<Arc<Self>>,
+ segment: Arc<[Message]>,
+ len: usize,
+}
+
+impl MessageHistory {
+ fn new(parent: Option<Arc<Self>>, segment: Arc<[Message]>) -> Self {
+ let len = parent.as_ref().map_or(0, |history| history.len) + segment.len();
+ Self {
+ parent,
+ segment,
+ len,
+ }
+ }
+
+ fn extend(&self, messages: &mut Vec<Message>) {
+ let mut segments = Vec::new();
+ let mut history = Some(self);
+ while let Some(node) = history {
+ segments.push(node.segment.as_ref());
+ history = node.parent.as_deref();
+ }
+ for segment in segments.into_iter().rev() {
+ messages.extend_from_slice(segment);
+ }
+ }
}
-/// Stored Responses IDs and conversation IDs, mapped to the model that served them.
+#[derive(Clone)]
+struct StoredState {
+ model: ModelId,
+ /// `None` means the provider owns the state and only routing must be pinned.
+ history: Option<Arc<MessageHistory>>,
+}
+
+#[derive(Clone)]
+struct CanonicalInput {
+ parent: Option<Arc<MessageHistory>>,
+ messages: Arc<[Message]>,
+}
+
+/// Stored Responses IDs and conversation IDs, with their provider or canonical state.
///
/// Keep earlier response IDs because a client can continue from any stored response.
/// Reject new IDs at capacity rather than forgetting where an earlier response is stored.
#[derive(Default)]
struct StateOwners {
- by_id: HashMap<String, ModelId>,
+ by_id: HashMap<String, StoredState>,
}
const MAX_STATE_OWNERS: usize = 65_536;
impl StateOwners {
- fn owner(&self, id: &str) -> Option<&ModelId> {
+ fn owner(&self, id: &str) -> Option<&StoredState> {
self.by_id.get(id)
}
@@ -433,7 +482,9 @@ impl StateOwners {
response_id: Option<&str>,
conversation_id: Option<&str>,
model: &ModelId,
+ history: Option<Arc<MessageHistory>>,
) -> std::result::Result<(), LlmClientError> {
+ let materialized = history.is_some();
let ids = [
response_id,
conversation_id.filter(|id| Some(*id) != response_id),
@@ -441,7 +492,9 @@ impl StateOwners {
let mut new_ids = 0;
for id in ids.into_iter().flatten() {
match self.by_id.get(id) {
- Some(owner) if owner != model => return Err(LlmClientError::ResponseStateConflict),
+ Some(owner) if owner.model != *model && !materialized => {
+ return Err(LlmClientError::ResponseStateConflict);
+ }
None => new_ids += 1,
_ => {}
}
@@ -451,11 +504,22 @@ impl StateOwners {
limit: MAX_STATE_OWNERS,
});
}
- for id in ids.into_iter().flatten() {
- if !self.by_id.contains_key(id) {
- self.by_id.insert(id.to_owned(), model.clone());
+ let state = StoredState {
+ model: model.clone(),
+ history,
+ };
+ if let Some(id) = response_id {
+ if materialized {
+ self.by_id.insert(id.to_owned(), state.clone());
+ } else {
+ self.by_id
+ .entry(id.to_owned())
+ .or_insert_with(|| state.clone());
}
}
+ if let Some(id) = conversation_id {
+ self.by_id.entry(id.to_owned()).or_insert(state);
+ }
Ok(())
}
}
@@ -485,9 +549,10 @@ struct ClientRouting {
routing: Routing,
target_prompts: HashMap<ModelId, String>,
routing_answer_target: Option<ModelId>,
- /// Track state only when completion targets use different clients. A separate
- /// classifier client does not require tracking when all completion targets share a client.
- state_owners: Option<Mutex<StateOwners>>,
+ /// Provider-owned routing pins and materialized cross-format history.
+ state_owners: Mutex<StateOwners>,
+ /// Native Responses state only needs a routing pin when answer targets span clients.
+ track_provider_state: bool,
}
enum Routing {
@@ -528,9 +593,9 @@ impl ClientRouter {
/// Build a router with an explicit list of models that can answer requests.
///
/// `by_model` contains every callable model, including classifiers. Only the model IDs in
- /// `completion_targets` determine whether completion clients differ and Responses state
- /// needs tracking. Classifier-only models remain callable without affecting that comparison.
- /// `target_prompts` and `routing_answer_target` have the same meaning as in
+ /// `completion_targets` determine whether native Responses state needs a routing pin.
+ /// Cross-format state is recorded lazily when a Responses request uses a Chat or Anthropic
+ /// target. `target_prompts` and `routing_answer_target` have the same meaning as in
/// [`Self::new_with_target_prompts`].
pub fn new_with_completion_targets(
by_model: HashMap<ModelId, Arc<dyn RoutedLlmClient>>,
@@ -549,7 +614,8 @@ impl ClientRouter {
routing: Routing::ByModel(by_model),
target_prompts,
routing_answer_target,
- state_owners: spans_providers.then(Mutex::default),
+ state_owners: Mutex::default(),
+ track_provider_state: spans_providers,
}),
}
}
@@ -565,7 +631,8 @@ impl ClientRouter {
routing: Routing::Single(client),
target_prompts: HashMap::new(),
routing_answer_target: None,
- state_owners: None,
+ state_owners: Mutex::default(),
+ track_provider_state: false,
}),
}
}
@@ -592,36 +659,67 @@ impl ClientRouter {
/// Return the recorded model for the requested response or conversation ID.
fn stored_state_owner(&self, request: &Request) -> Option<StateOwner> {
- let owners = self.inner.state_owners.as_ref()?;
let fields = &request.llm_request.extensions.fields;
let (trigger, id) = match fields.get("previous_response_id").and_then(Value::as_str) {
Some(id) => ("previous_response_id", id),
None => ("conversation", conversation_id(fields)?),
};
- let model = owners.lock().owner(id)?.clone();
- Some(StateOwner { model, trigger })
+ let state = self.inner.state_owners.lock().owner(id)?.clone();
+ Some(StateOwner {
+ model: state.model,
+ trigger,
+ history: state.history,
+ })
}
- /// Record state reported by a Responses provider. Other response formats have IDs
- /// that cannot be continued through the Responses API.
- fn remember_state_owner(&self, request: &Request, mut response: Response) -> Result<Response> {
- if self.inner.state_owners.is_none() {
- return Ok(response);
+ fn canonical_input(&self, request: &Request) -> CanonicalInput {
+ let parent = request
+ .llm_request
+ .extensions
+ .fields
+ .get("previous_response_id")
+ .and_then(Value::as_str)
+ .and_then(|id| self.inner.state_owners.lock().owner(id)?.history.clone());
+ let parent_len = parent.as_ref().map_or(0, |history| history.len);
+ let (parent, messages) = match request.llm_request.messages.get(parent_len..) {
+ Some(messages) => (parent, messages.to_vec()),
+ None => (None, request.llm_request.messages.clone()),
+ };
+ CanonicalInput {
+ parent,
+ messages: Arc::from(messages),
}
+ }
+
+ /// Record provider-owned Responses state or canonical history for a cross-format response.
+ fn remember_state_owner(&self, request: &Request, mut response: Response) -> Result<Response> {
let Some(model) = response.served_model().cloned() else {
return Ok(response);
};
let fields = &request.llm_request.extensions.fields;
let store = fields.get("store").and_then(Value::as_bool) != Some(false);
let conversation = conversation_id(fields).map(str::to_owned);
+ let responses_format = WireFormat::OpenAiResponses.into();
+ let responses_request = request
+ .llm_request
+ .preservation
+ .requests
+ .contains_key(&responses_format)
+ || fields.contains_key("previous_response_id")
+ || fields.contains_key("conversation");
+ if !responses_request && !self.inner.track_provider_state {
+ return Ok(response);
+ }
+ let canonical_input = responses_request.then(|| self.canonical_input(request));
response.llm_response = match response.llm_response {
LlmResponse::Agg(agg) => {
- if let Some(body) = agg
- .preservation
- .responses
- .get(&WireFormat::OpenAiResponses.into())
- {
- self.remember_response(body, &model, store, conversation.as_deref())
+ if let Some(body) = agg.preservation.responses.get(&responses_format) {
+ if self.inner.track_provider_state {
+ self.remember_response(body, &model, store, conversation.as_deref())
+ .map_err(|error| LibsyError::client_call(model.clone(), error))?;
+ }
+ } else if let Some(input) = &canonical_input {
+ self.remember_canonical_response(&agg, &model, store, input)
.map_err(|error| LibsyError::client_call(model.clone(), error))?;
}
LlmResponse::Agg(agg)
@@ -629,24 +727,82 @@ impl ClientRouter {
LlmResponse::Stream(stream) => {
let router = self.clone();
LlmResponse::Stream(Box::pin(futures_util::stream::try_unfold(
- (stream, router, model, conversation),
- move |(mut stream, router, model, conversation)| async move {
+ (
+ stream,
+ router,
+ model,
+ conversation,
+ canonical_input,
+ ResponseAccumulator::new(),
+ false,
+ true,
+ ),
+ move |(
+ mut stream,
+ router,
+ model,
+ conversation,
+ canonical_input,
+ mut accumulator,
+ mut native_responses,
+ mut valid,
+ )| async move {
let Some(event) = stream.next().await else {
+ if let Some(input) = &canonical_input
+ && !native_responses
+ && valid
+ {
+ router.remember_canonical_response(
+ &accumulator.finish(),
+ &model,
+ store,
+ input,
+ )?;
+ }
return Ok(None);
};
let event = event?;
if let Some(preserved) = event.preservation()
&& preserved.source().as_str() == WireFormat::OpenAiResponses.as_str()
- && let Some(body) = preserved.raw().get("response")
{
- router.remember_response(
- body,
- &model,
- store,
- conversation.as_deref(),
- )?;
+ native_responses = true;
+ if router.inner.track_provider_state
+ && let Some(body) = preserved.raw().get("response")
+ {
+ router.remember_response(
+ body,
+ &model,
+ store,
+ conversation.as_deref(),
+ )?;
+ }
}
- Ok(Some((event, (stream, router, model, conversation))))
+ if responses_request {
+ for chunk in event.normalized() {
+ if matches!(
+ chunk,
+ LlmResponseChunk::DecodeError { .. }
+ | LlmResponseChunk::StreamError { .. }
+ ) {
+ valid = false;
+ } else {
+ accumulator.push(chunk.clone());
+ }
+ }
+ }
+ Ok(Some((
+ event,
+ (
+ stream,
+ router,
+ model,
+ conversation,
+ canonical_input,
+ accumulator,
+ native_responses,
+ valid,
+ ),
+ )))
},
)))
}
@@ -663,21 +819,59 @@ impl ClientRouter {
store: bool,
conversation: Option<&str>,
) -> std::result::Result<(), LlmClientError> {
- if let Some(owners) = &self.inner.state_owners {
- let mut owners = owners.lock();
- let response_id = body
- .get("id")
- .and_then(Value::as_str)
- .filter(|_| body.get("store").and_then(Value::as_bool).unwrap_or(store));
- let conversation = body.as_object().and_then(conversation_id).or(conversation);
- if let Err(error) = owners.remember(response_id, conversation, model) {
- tracing::error!(error = %error, "could not record Responses state owner");
+ let mut owners = self.inner.state_owners.lock();
+ let response_id = body
+ .get("id")
+ .and_then(Value::as_str)
+ .filter(|_| body.get("store").and_then(Value::as_bool).unwrap_or(store));
+ let conversation = body.as_object().and_then(conversation_id).or(conversation);
+ if let Err(error) = owners.remember(response_id, conversation, model, None) {
+ if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
+ tracing::warn!(%error, "Responses state owner capacity reached; state was not retained");
+ } else {
+ tracing::error!(%error, "could not record Responses state owner");
return Err(error);
}
}
Ok(())
}
+ fn remember_canonical_response(
+ &self,
+ response: &AggLlmResponse,
+ model: &ModelId,
+ store: bool,
+ input: &CanonicalInput,
+ ) -> std::result::Result<(), LlmClientError> {
+ let response_id = response.id.as_deref().filter(|_| store);
+ if response_id.is_none() {
+ return Ok(());
+ }
+ let mut segment = input.messages.to_vec();
+ segment.extend(response.outputs.iter().map(|output| Message {
+ role: output.role,
+ content: output.content.clone(),
+ }));
+ let history = Arc::new(MessageHistory::new(
+ input.parent.clone(),
+ Arc::from(segment),
+ ));
+ let result =
+ self.inner
+ .state_owners
+ .lock()
+ .remember(response_id, None, model, Some(history));
+ if let Err(error) = result {
+ if matches!(error, LlmClientError::ResponseStateLimitExceeded { .. }) {
+ tracing::warn!(%error, "cross-format Responses state capacity reached; history was not retained");
+ return Ok(());
+ }
+ tracing::error!(%error, "could not record cross-format Responses state");
+ return Err(error);
+ }
+ Ok(())
+ }
+
/// Prepare a completion candidate with its configured target prompt.
fn prepare_completion_request(&self, mut request: Request, target: &ModelId) -> Request {
let prompt = self.inner.target_prompts.get(target).map(String::as_str);
@@ -713,8 +907,8 @@ mod tests {
use http::StatusCode;
use switchyard_libsy::{Driver, RoutingOutcome};
use switchyard_protocol::{
- ContentBlock, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, completion_text,
- text_request, text_response,
+ ContentBlock, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Message, Role,
+ ToolResult, completion_text, text_request, text_response,
};
use wiremock::matchers::method;
use wiremock::{Mock, MockServer, ResponseTemplate};
@@ -1089,33 +1283,428 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn responses_stored_tool_continuation_materializes_for_anthropic() -> Result<()> {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .respond_with(|request: &wiremock::Request| {
+ let body: Value = serde_json::from_slice(&request.body).expect("request JSON");
+ let blocks = body["messages"]
+ .as_array()
+ .into_iter()
+ .flatten()
+ .filter_map(|message| message["content"].as_array())
+ .flatten()
+ .collect::<Vec<_>>();
+ let has_result = blocks.iter().any(|block| block["type"] == "tool_result");
+ let has_call = blocks.iter().any(|block| block["type"] == "tool_use");
+ let has_thanks = blocks
+ .iter()
+ .any(|block| block["type"] == "text" && block["text"] == "thanks")
+ || body["messages"].as_array().is_some_and(|messages| {
+ messages
+ .iter()
+ .any(|message| message["content"] == "thanks")
+ });
+ if has_result && !has_call {
+ return ResponseTemplate::new(400).set_body_json(json!({
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "message": "tool_result has no corresponding tool_use"
+ }
+ }));
+ }
+ let response = if has_thanks {
+ json!({
+ "id": "msg_third", "type": "message", "role": "assistant",
+ "model": "weak", "content": [{"type": "text", "text": "done"}],
+ "stop_reason": "end_turn", "usage": {"input_tokens": 6, "output_tokens": 1}
+ })
+ } else if has_result {
+ json!({
+ "id": "msg_follow", "type": "message", "role": "assistant",
+ "model": "weak", "content": [{"type": "text", "text": "sunny"}],
+ "stop_reason": "end_turn", "usage": {"input_tokens": 4, "output_tokens": 1}
+ })
+ } else {
+ json!({
+ "id": "msg_seed", "type": "message", "role": "assistant",
+ "model": "weak",
+ "content": [{
+ "type": "tool_use", "id": "toolu_weather", "name": "get_weather",
+ "input": {"city": "Paris"}
+ }],
+ "stop_reason": "tool_use", "usage": {"input_tokens": 3, "output_tokens": 1}
+ })
+ };
+ ResponseTemplate::new(200).set_body_json(response)
+ })
+ .mount(&server)
+ .await;
+
+ let client: Arc<dyn RoutedLlmClient> = Arc::new(
+ TranslatingLlmClient::new(&[ModelConfig::new(
+ "weak",
+ Backend::Anthropic(HttpBackendConfig {
+ base_url: server.uri(),
+ api_key: None,
+ forward_auth: false,
+ extra_headers: BTreeMap::new(),
+ extra_body: BTreeMap::new(),
+ reasoning_effort: None,
+ max_retries: 0,
+ timeout: None,
+ }),
+ None,
+ )])
+ .map_err(|error| LibsyError::external("building test client", error))?,
+ );
+ let clients = ClientRouter::new(HashMap::from([(ModelId::from("weak"), client)]));
+ let models = to_category_map(&["weak"]);
+
+ let seed = Request {
+ llm_request: switchyard_translation::decode_request(
+ WireFormat::OpenAiResponses,
+ &json!({
+ "model": "route",
+ "input": "Call get_weather for Paris",
+ "tools": [{
+ "type": "function", "name": "get_weather",
+ "parameters": {"type": "object"}
+ }],
+ "store": true
+ }),
+ )
+ .map_err(|error| LibsyError::external("decoding seed request", error))?,
+ raw_request: None,
+ metadata: None,
+ };
+ let (_, response) = run(
+ Arc::new(switchyard_libsy::Passthrough),
+ clients.clone(),
+ seed,
+ models.clone(),
+ None,
+ )
+ .await?;
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .and_then(|agg| agg.id.as_deref()),
+ Some("msg_seed")
+ );
+
+ let follow = Request {
+ llm_request: switchyard_translation::decode_request(
+ WireFormat::OpenAiResponses,
+ &json!({
+ "model": "route",
+ "previous_response_id": "msg_seed",
+ "input": [{
+ "type": "function_call_output",
+ "call_id": "toolu_weather",
+ "output": "Paris is sunny"
+ }],
+ "tools": [{
+ "type": "function", "name": "get_weather",
+ "parameters": {"type": "object"}
+ }],
+ "store": true
+ }),
+ )
+ .map_err(|error| LibsyError::external("decoding continuation request", error))?,
+ raw_request: None,
+ metadata: None,
+ };
+ let (_, response) = run(
+ Arc::new(switchyard_libsy::Passthrough),
+ clients.clone(),
+ follow,
+ models.clone(),
+ None,
+ )
+ .await?;
+ assert_eq!(
+ completion_text(
+ response
+ .llm_response
+ .as_agg()
+ .expect("buffered Anthropic response"),
+ ),
+ "sunny"
+ );
+
+ let third = Request {
+ llm_request: switchyard_translation::decode_request(
+ WireFormat::OpenAiResponses,
+ &json!({
+ "model": "route",
+ "previous_response_id": "msg_follow",
+ "input": "thanks",
+ "store": true
+ }),
+ )
+ .map_err(|error| LibsyError::external("decoding chained request", error))?,
+ raw_request: None,
+ metadata: None,
+ };
+ let history = clients
+ .stored_state_owner(&third)
+ .and_then(|owner| owner.history)
+ .expect("chained canonical history");
+ assert_eq!(history.len, 4);
+ assert_eq!(history.segment.len(), 2);
+ assert!(history.parent.is_some());
+ let (_, response) = run(
+ Arc::new(switchyard_libsy::Passthrough),
+ clients,
+ third,
+ models,
+ None,
+ )
+ .await?;
+ assert_eq!(
+ completion_text(
+ response
+ .llm_response
+ .as_agg()
+ .expect("buffered Anthropic response"),
+ ),
+ "done"
+ );
+
+ let requests = server.received_requests().await.expect("request recording");
+ assert_eq!(requests.len(), 3);
+ let follow: Value = serde_json::from_slice(&requests[1].body)
+ .map_err(|error| LibsyError::external("decoding captured request", error))?;
+ assert_eq!(follow["messages"][1]["content"][0]["type"], "tool_use");
+ assert_eq!(follow["messages"][2]["content"][0]["type"], "tool_result");
+ let third: Value = serde_json::from_slice(&requests[2].body)
+ .map_err(|error| LibsyError::external("decoding captured request", error))?;
+ assert_eq!(third["messages"][1]["content"][0]["type"], "tool_use");
+ assert_eq!(third["messages"][2]["content"][0]["type"], "tool_result");
+ assert_eq!(third["messages"][3]["content"], "sunny");
+ assert_eq!(third["messages"][4]["content"][0]["text"], "thanks");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn streamed_cross_format_state_is_recorded_only_after_completion() -> Result<()> {
+ let client = Arc::new(CandidateClient {
+ calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
+ first: FirstOutcome::StreamSuccess,
+ });
+ let clients = ClientRouter::new(HashMap::from([(
+ ModelId::from("weak"),
+ client as Arc<dyn RoutedLlmClient>,
+ )]));
+ let mut seed = request();
+ seed.llm_request.messages = vec![Message::text(Role::User, "inspect")];
+ seed.llm_request
+ .preservation
+ .requests
+ .insert(WireFormat::OpenAiResponses.into(), json!({}));
+ let mut response = stream_response(vec![
+ LlmResponseChunk::MessageStart {
+ id: Some("msg_stream".to_string()),
+ model: Some("weak".to_string()),
+ },
+ LlmResponseChunk::ToolCallDelta {
+ index: 0,
+ id: Some("toolu_stream".to_string()),
+ name: Some("lookup".to_string()),
+ arguments_delta: Some("{}".to_string()),
+ },
+ LlmResponseChunk::MessageStop {
+ reason: Some("tool_use".to_string()),
+ },
+ ]);
+ response.set_served_model(&ModelId::from("weak"));
+ let response = clients.remember_state_owner(&seed, response)?;
+
+ let mut follow = request();
+ follow
+ .llm_request
+ .extensions
+ .fields
+ .insert("previous_response_id".to_string(), json!("msg_stream"));
+ follow.llm_request.messages = vec![Message {
+ role: Role::Tool,
+ content: vec![ContentBlock::ToolResult(ToolResult {
+ tool_call_id: "toolu_stream".to_string(),
+ content: vec![ContentBlock::Text {
+ text: "ready".to_string(),
+ }],
+ is_error: None,
+ })],
+ }];
+ assert!(clients.stored_state_owner(&follow).is_none());
+ response
+ .llm_response
+ .into_agg()
+ .await
+ .map_err(|error| LibsyError::client_call("weak", error))?;
+
+ let outcome = continue_on(
+ clients
+ .stored_state_owner(&follow)
+ .expect("completed stream state"),
+ "passthrough",
+ follow,
+ );
+ assert_eq!(outcome.request.llm_request.messages.len(), 3);
+ assert!(matches!(
+ outcome.request.llm_request.messages[1].content.as_slice(),
+ [ContentBlock::ToolCall(call)] if call.id == "toolu_stream"
+ ));
+ assert!(matches!(
+ outcome.request.llm_request.messages[2].content.as_slice(),
+ [ContentBlock::ToolResult(result)] if result.tool_call_id == "toolu_stream"
+ ));
+ Ok(())
+ }
+
+ #[test]
+ fn cross_format_response_with_store_false_is_not_retained() -> Result<()> {
+ let client = Arc::new(CandidateClient {
+ calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
+ first: FirstOutcome::StreamSuccess,
+ });
+ let clients = ClientRouter::new(HashMap::from([(
+ ModelId::from("weak"),
+ client as Arc<dyn RoutedLlmClient>,
+ )]));
+ let mut seed = request();
+ seed.llm_request
+ .extensions
+ .fields
+ .insert("store".to_string(), json!(false));
+ seed.llm_request
+ .preservation
+ .requests
+ .insert(WireFormat::OpenAiResponses.into(), json!({}));
+ let mut agg = text_response(Some("weak".to_string()), "done".to_string());
+ agg.id = Some("msg_not_stored".to_string());
+ let mut response = Response {
+ llm_response: LlmResponse::Agg(agg),
+ metadata: None,
+ upstream_headers: http::HeaderMap::new(),
+ };
+ response.set_served_model(&ModelId::from("weak"));
+ clients.remember_state_owner(&seed, response)?;
+
+ let mut follow = request();
+ follow
+ .llm_request
+ .extensions
+ .fields
+ .insert("previous_response_id".to_string(), json!("msg_not_stored"));
+ assert!(clients.stored_state_owner(&follow).is_none());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn state_capacity_does_not_fail_completed_responses() -> Result<()> {
+ let client = Arc::new(CandidateClient {
+ calls: Mutex::new(Vec::new()),
+ requests: Mutex::new(Vec::new()),
+ first: FirstOutcome::StreamSuccess,
+ });
+ let clients = ClientRouter::new(HashMap::from([(
+ ModelId::from("weak"),
+ client as Arc<dyn RoutedLlmClient>,
+ )]));
+ let model = ModelId::from("weak");
+ clients.inner.state_owners.lock().by_id = (0..MAX_STATE_OWNERS)
+ .map(|id| {
+ (
+ format!("existing_{id}"),
+ StoredState {
+ model: model.clone(),
+ history: None,
+ },
+ )
+ })
+ .collect();
+
+ let mut request = request();
+ request
+ .llm_request
+ .preservation
+ .requests
+ .insert(WireFormat::OpenAiResponses.into(), json!({}));
+ let mut agg = text_response(Some("weak".to_string()), "done".to_string());
+ agg.id = Some("overflow_buffered".to_string());
+ let mut buffered = Response {
+ llm_response: LlmResponse::Agg(agg),
+ metadata: None,
+ upstream_headers: http::HeaderMap::new(),
+ };
+ buffered.set_served_model(&model);
+ clients.remember_state_owner(&request, buffered)?;
+
+ let mut streamed = stream_response(vec![
+ LlmResponseChunk::MessageStart {
+ id: Some("overflow_streamed".to_string()),
+ model: Some("weak".to_string()),
+ },
+ LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "done".to_string(),
+ },
+ LlmResponseChunk::MessageStop {
+ reason: Some("stop".to_string()),
+ },
+ ]);
+ streamed.set_served_model(&model);
+ clients
+ .remember_state_owner(&request, streamed)?
+ .llm_response
+ .into_agg()
+ .await
+ .map_err(|error| LibsyError::client_call("weak", error))?;
+
+ assert_eq!(
+ clients.inner.state_owners.lock().by_id.len(),
+ MAX_STATE_OWNERS
+ );
+ Ok(())
+ }
+
#[test]
fn state_owners_preserve_existing_records_at_capacity()
-> std::result::Result<(), LlmClientError> {
let mut owners = StateOwners::default();
let model = ModelId::from("model/a");
for id in 0..MAX_STATE_OWNERS - 1 {
- owners.remember(Some(&format!("resp_{id}")), None, &model)?;
+ owners.remember(Some(&format!("resp_{id}")), None, &model, None)?;
}
assert!(matches!(
- owners.remember(Some("new_response"), Some("new_conversation"), &model),
+ owners.remember(Some("new_response"), Some("new_conversation"), &model, None,),
Err(LlmClientError::ResponseStateLimitExceeded {
limit: MAX_STATE_OWNERS
})
));
assert!(owners.owner("new_response").is_none());
assert!(owners.owner("new_conversation").is_none());
- owners.remember(Some("last_id"), Some("last_id"), &model)?;
- owners.remember(Some("last_id"), Some("resp_0"), &model)?;
+ owners.remember(Some("last_id"), Some("last_id"), &model, None)?;
+ owners.remember(Some("last_id"), Some("resp_0"), &model, None)?;
assert!(matches!(
- owners.remember(Some("over_limit"), None, &model),
+ owners.remember(Some("over_limit"), None, &model, None),
Err(LlmClientError::ResponseStateLimitExceeded { .. })
));
assert!(matches!(
- owners.remember(Some("resp_0"), None, &ModelId::from("model/b")),
+ owners.remember(Some("resp_0"), None, &ModelId::from("model/b"), None,),
Err(LlmClientError::ResponseStateConflict)
));
- assert_eq!(owners.owner("resp_0"), Some(&model));
+ assert_eq!(
+ owners.owner("resp_0").map(|state| &state.model),
+ Some(&model)
+ );
assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
Ok(())
}
@@ -1125,7 +1714,15 @@ mod tests {
let model = ModelId::from("model/a");
let owners = Arc::new(Mutex::new(StateOwners {
by_id: (0..MAX_STATE_OWNERS - 16)
- .map(|id| (format!("existing_{id}"), model.clone()))
+ .map(|id| {
+ (
+ format!("existing_{id}"),
+ StoredState {
+ model: model.clone(),
+ history: None,
+ },
+ )
+ })
.collect(),
}));
let barrier = Arc::new(std::sync::Barrier::new(32));
@@ -1141,6 +1738,7 @@ mod tests {
Some(&format!("resp_{id}")),
Some(&format!("conv_{id}")),
&model,
+ None,
);
match result {
Ok(()) => Some(id),
@@ -1158,10 +1756,23 @@ mod tests {
let owners = owners.lock();
assert_eq!(accepted.len(), 8);
assert_eq!(owners.by_id.len(), MAX_STATE_OWNERS);
- assert_eq!(owners.owner("existing_0"), Some(&model));
+ assert_eq!(
+ owners.owner("existing_0").map(|state| &state.model),
+ Some(&model)
+ );
for id in accepted {
- assert_eq!(owners.owner(&format!("resp_{id}")), Some(&model));
- assert_eq!(owners.owner(&format!("conv_{id}")), Some(&model));
+ assert_eq!(
+ owners
+ .owner(&format!("resp_{id}"))
+ .map(|state| &state.model),
+ Some(&model)
+ );
+ assert_eq!(
+ owners
+ .owner(&format!("conv_{id}"))
+ .map(|state| &state.model),
+ Some(&model)
+ );
}
}