Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

agent: Share compatible Chat Completions infrastructure (#62652)

Anant Goel9f164a0d2eec
diff --git a/Cargo.lock b/Cargo.lock
index ec12c526500..5eda1ece25f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9963,6 +9963,7 @@ dependencies = [
"serde",
"serde_json",
"settings",
+ "sha2",
"strum 0.27.2",
"tokio",
"ui",
@@ -12521,6 +12522,7 @@ dependencies = [
"futures 0.3.32",
"http_client",
"language_model_core",
+ "open_ai",
"schemars 1.0.4",
"serde",
"serde_json",
diff --git a/crates/edit_prediction_cli/src/openai_client.rs b/crates/edit_prediction_cli/src/openai_client.rs
index 72b8e75e864..b1bb44fa409 100644
--- a/crates/edit_prediction_cli/src/openai_client.rs
+++ b/crates/edit_prediction_cli/src/openai_client.rs
@@ -488,6 +488,7 @@ impl BatchingOpenAiClient {
content: Some(MessageContent::Plain(msg.content)),
tool_calls: Vec::new(),
reasoning_content: None,
+ reasoning_details: None,
},
"system" => RequestMessage::System {
content: MessageContent::Plain(msg.content),
diff --git a/crates/edit_prediction_cli/src/repair.rs b/crates/edit_prediction_cli/src/repair.rs
index a6f82bc6453..8034c206a3a 100644
--- a/crates/edit_prediction_cli/src/repair.rs
+++ b/crates/edit_prediction_cli/src/repair.rs
@@ -389,6 +389,7 @@ pub async fn run_repair(
content: Some(open_ai::MessageContent::Plain(teacher_response.clone())),
tool_calls: vec![],
reasoning_content: None,
+ reasoning_details: None,
},
// Turn 3: Repair critique and instructions
open_ai::RequestMessage::User {
diff --git a/crates/language_models/Cargo.toml b/crates/language_models/Cargo.toml
index eb9eb03d3eb..4c2a7d1bc49 100644
--- a/crates/language_models/Cargo.toml
+++ b/crates/language_models/Cargo.toml
@@ -59,6 +59,7 @@ schemars.workspace = true
serde.workspace = true
serde_json.workspace = true
settings.workspace = true
+sha2.workspace = true
strum.workspace = true
tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }
ui.workspace = true
diff --git a/crates/language_models/src/provider/bedrock.rs b/crates/language_models/src/provider/bedrock.rs
index 5601ee906f7..b4b503076d7 100644
--- a/crates/language_models/src/provider/bedrock.rs
+++ b/crates/language_models/src/provider/bedrock.rs
@@ -1270,7 +1270,7 @@ where
provider: provider_name.to_owned(),
status_code: response.status(),
body,
- headers: response.headers().clone(),
+ headers: Box::new(response.headers().clone()),
})
}
}
diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rs
index 59d4aac3fb7..8b8ce92e771 100644
--- a/crates/language_models/src/provider/open_router.rs
+++ b/crates/language_models/src/provider/open_router.rs
@@ -12,10 +12,12 @@ use language_model::{
LanguageModelToolResultContent, LanguageModelToolSchemaFormat, LanguageModelToolUse,
MessageContent, ProviderSettingsView, RateLimiter, Role, StopReason, TokenUsage, env_var,
};
+use open_ai::completion::ReasoningDetailsAccumulator;
use open_router::{
Model, ModelMode as OpenRouterModelMode, OPEN_ROUTER_API_URL, ResponseStreamEvent, list_models,
};
use settings::{OpenRouterAvailableModel as AvailableModel, Settings, SettingsStore};
+use sha2::{Digest as _, Sha256};
use std::pin::Pin;
use std::sync::{Arc, LazyLock};
use ui::IconName;
@@ -28,7 +30,6 @@ const PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderName::new(
const API_KEY_ENV_VAR_NAME: &str = "OPENROUTER_API_KEY";
static API_KEY_ENV_VAR: LazyLock<EnvVar> = env_var!(API_KEY_ENV_VAR_NAME);
pub(crate) const RESERVED_HEADER_NAMES: &[&str] = &["HTTP-Referer", "X-Title"];
-const MAX_OPEN_ROUTER_SESSION_ID_LENGTH: usize = 256;
#[derive(Default, Clone, Debug, PartialEq)]
pub struct OpenRouterSettings {
@@ -620,10 +621,10 @@ pub fn into_open_router(
fn open_router_session_id(thread_id: Option<String>) -> Option<String> {
thread_id.map(|thread_id| {
- thread_id
- .chars()
- .take(MAX_OPEN_ROUTER_SESSION_ID_LENGTH)
- .collect()
+ let mut hasher = Sha256::new();
+ hasher.update(b"zed-openrouter-session-v1\0");
+ hasher.update(thread_id.as_bytes());
+ format!("{:x}", hasher.finalize())
})
}
@@ -717,14 +718,14 @@ fn add_message_content_part(
pub struct OpenRouterEventMapper {
tool_calls_by_index: HashMap<usize, RawToolCall>,
- reasoning_details: Option<serde_json::Value>,
+ reasoning_details: ReasoningDetailsAccumulator,
}
impl OpenRouterEventMapper {
pub fn new() -> Self {
Self {
tool_calls_by_index: HashMap::default(),
- reasoning_details: None,
+ reasoning_details: ReasoningDetailsAccumulator::default(),
}
}
@@ -776,12 +777,10 @@ impl OpenRouterEventMapper {
return events;
};
- if let Some(details) = choice.delta.reasoning_details.clone() {
- // Emit reasoning_details immediately
- events.push(Ok(LanguageModelCompletionEvent::ReasoningDetails(
- details.clone(),
- )));
- self.reasoning_details = Some(details);
+ if let Some(details) = choice.delta.reasoning_details.clone()
+ && let Some(details) = self.reasoning_details.push(details)
+ {
+ events.push(Ok(LanguageModelCompletionEvent::ReasoningDetails(details)));
}
if let Some(reasoning) = choice.delta.reasoning.clone() {
@@ -842,7 +841,6 @@ impl OpenRouterEventMapper {
match choice.finish_reason.as_deref() {
Some("stop") => {
- // Don't emit reasoning_details here - already emitted immediately when captured
events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)));
}
Some("tool_calls") => {
@@ -867,12 +865,10 @@ impl OpenRouterEventMapper {
}
}));
- // Don't emit reasoning_details here - already emitted immediately when captured
events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse)));
}
Some(stop_reason) => {
log::error!("Unexpected OpenRouter stop_reason: {stop_reason:?}",);
- // Don't emit reasoning_details here - already emitted immediately when captured
events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)));
}
None => {}
@@ -1036,46 +1032,21 @@ mod tests {
}
}
- // Assertions
assert!(has_tool_use, "Should have emitted ToolUse event");
- assert!(
- !reasoning_details_events.is_empty(),
- "Should have emitted ReasoningDetails events"
- );
-
- // We should have received multiple reasoning_details events (text, encrypted, empty)
- // The agent layer is responsible for keeping only the first non-empty one
- assert!(
- reasoning_details_events.len() >= 2,
- "Should have multiple reasoning_details events from streaming"
- );
-
- // Verify at least one contains the encrypted data
- let has_encrypted = reasoning_details_events.iter().any(|details| {
- if let serde_json::Value::Array(arr) = details {
- arr.iter().any(|item| {
- item["type"] == "reasoning.encrypted"
- && item["data"]
- .as_str()
- .map_or(false, |s| s.contains("EtgDCtUDAdHtim9OF5jm4aeZSBAtl"))
- })
- } else {
- false
- }
- });
- assert!(
- has_encrypted,
- "Should have at least one reasoning_details with encrypted data"
- );
-
- // Verify thought_signature was captured
- assert!(
- thought_signature_value.is_some(),
- "Tool use should have thought_signature"
+ assert_eq!(reasoning_details_events.len(), 2);
+ let final_details = reasoning_details_events
+ .last()
+ .and_then(serde_json::Value::as_array)
+ .and_then(|details| details.first())
+ .expect("accumulated reasoning details");
+ assert_eq!(final_details["text"], "Let me analyze this request...");
+ assert_eq!(
+ final_details["data"],
+ "EtgDCtUDAdHtim9OF5jm4aeZSBAtl/randomized123"
);
assert_eq!(
- thought_signature_value.unwrap(),
- "sha256:test_signature_xyz789"
+ thought_signature_value.as_deref(),
+ Some("sha256:test_signature_xyz789")
);
}
@@ -1113,7 +1084,7 @@ mod tests {
}
#[gpui::test]
- async fn test_session_id_uses_thread_id() {
+ async fn test_session_id_is_stable_without_exposing_thread_id() {
let model = open_router::Model::new(
"openai/gpt-4o",
Some("GPT-4o"),
@@ -1123,9 +1094,9 @@ mod tests {
None,
None,
);
- let expected_session_id = "a".repeat(MAX_OPEN_ROUTER_SESSION_ID_LENGTH);
+ let thread_id = "internal-thread-id";
let request = LanguageModelRequest {
- thread_id: Some(format!("{expected_session_id}extra")),
+ thread_id: Some(thread_id.to_string()),
messages: vec![language_model::LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Hello".to_string())],
@@ -1138,51 +1109,14 @@ mod tests {
let result = into_open_router(request, &model, None).unwrap();
assert_eq!(
- result.session_id.as_deref(),
- Some(expected_session_id.as_str())
+ result.session_id,
+ open_router_session_id(Some(thread_id.into()))
+ );
+ assert_ne!(result.session_id.as_deref(), Some(thread_id));
+ assert_ne!(
+ result.session_id,
+ open_router_session_id(Some("another-thread-id".into()))
);
- }
-
- #[gpui::test]
- async fn test_agent_prevents_empty_reasoning_details_overwrite() {
- // This test verifies that the agent layer prevents empty reasoning_details
- // from overwriting non-empty ones, even though the mapper emits all events.
-
- // Simulate what the agent does when it receives multiple ReasoningDetails events
- let mut agent_reasoning_details: Option<serde_json::Value> = None;
-
- let events = vec![
- // First event: non-empty reasoning_details
- serde_json::json!([
- {
- "type": "reasoning.encrypted",
- "data": "real_data_here",
- "format": "google-gemini-v1"
- }
- ]),
- // Second event: empty array (should not overwrite)
- serde_json::json!([]),
- ];
-
- for details in events {
- // This mimics the agent's logic: only store if we don't already have it
- if agent_reasoning_details.is_none() {
- agent_reasoning_details = Some(details);
- }
- }
-
- // Verify the agent kept the first non-empty reasoning_details
- assert!(agent_reasoning_details.is_some());
- let final_details = agent_reasoning_details.unwrap();
- if let serde_json::Value::Array(arr) = &final_details {
- assert!(
- !arr.is_empty(),
- "Agent should have kept the non-empty reasoning_details"
- );
- assert_eq!(arr[0]["data"], "real_data_here");
- } else {
- panic!("Expected array");
- }
}
#[gpui::test]
diff --git a/crates/language_models/src/provider/vercel_ai_gateway.rs b/crates/language_models/src/provider/vercel_ai_gateway.rs
index 23868b93747..6115b760461 100644
--- a/crates/language_models/src/provider/vercel_ai_gateway.rs
+++ b/crates/language_models/src/provider/vercel_ai_gateway.rs
@@ -331,7 +331,7 @@ fn map_open_ai_error(error: open_ai::RequestError) -> LanguageModelCompletionErr
retry_after,
)
}
- open_ai::RequestError::Other(error) => LanguageModelCompletionError::Other(error),
+ error => error.into(),
}
}
diff --git a/crates/open_ai/src/batches.rs b/crates/open_ai/src/batches.rs
index a93fef675dd..5005c3816b8 100644
--- a/crates/open_ai/src/batches.rs
+++ b/crates/open_ai/src/batches.rs
@@ -182,7 +182,7 @@ pub async fn upload_batch_file(
provider: "openai".to_owned(),
status_code: response.status(),
body,
- headers: response.headers().clone(),
+ headers: Box::new(response.headers().clone()),
})
}
}
@@ -229,7 +229,7 @@ pub async fn create_batch(
provider: "openai".to_owned(),
status_code: response.status(),
body,
- headers: response.headers().clone(),
+ headers: Box::new(response.headers().clone()),
})
}
}
@@ -273,7 +273,7 @@ pub async fn retrieve_batch(
provider: "openai".to_owned(),
status_code: response.status(),
body,
- headers: response.headers().clone(),
+ headers: Box::new(response.headers().clone()),
})
}
}
@@ -317,7 +317,7 @@ pub async fn download_file(
provider: "openai".to_owned(),
status_code: response.status(),
body,
- headers: response.headers().clone(),
+ headers: Box::new(response.headers().clone()),
})
}
}
diff --git a/crates/open_ai/src/chat_completion_transport_tests.rs b/crates/open_ai/src/chat_completion_transport_tests.rs
new file mode 100644
index 00000000000..e35a2c6124b
--- /dev/null
+++ b/crates/open_ai/src/chat_completion_transport_tests.rs
@@ -0,0 +1,206 @@
+use super::*;
+use futures::executor::block_on;
+use http_client::{
+ FakeHttpClient, Response,
+ http::{HeaderName, HeaderValue},
+};
+use serde_json::json;
+use std::sync::{Arc, Mutex};
+
+#[test]
+fn streaming_transport_serializes_custom_requests_and_reports_done() {
+ let captured_request = Arc::new(Mutex::new(None));
+ let captured_request_for_handler = captured_request.clone();
+ let client = FakeHttpClient::create(move |mut request| {
+ let captured_request = captured_request_for_handler.clone();
+ async move {
+ let uri = request.uri().to_string();
+ let authorization = request.headers()["authorization"]
+ .to_str()
+ .expect("valid authorization header")
+ .to_string();
+ let custom_header = request.headers()["x-compatible-provider"]
+ .to_str()
+ .expect("valid custom header")
+ .to_string();
+ let mut body = String::new();
+ request.body_mut().read_to_string(&mut body).await?;
+ captured_request
+ .lock()
+ .expect("captured request lock")
+ .replace((uri, authorization, custom_header, body));
+ Ok(Response::builder()
+ .status(200)
+ .body(AsyncBody::from(concat!(
+ ": keepalive\n",
+ "event: message\n",
+ "data:{\"chunk\":1}\n\n",
+ "data: [DONE]\n\n"
+ )))?)
+ }
+ });
+ let extra_headers = CustomHeaders::new(vec![(
+ HeaderName::from_static("x-compatible-provider"),
+ HeaderValue::from_static("enabled"),
+ )]);
+
+ let events = block_on(async {
+ stream_chat_completion(
+ client.as_ref(),
+ "Compatible Provider",
+ "https://example.com/v1",
+ " secret ",
+ &extra_headers,
+ &json!({"model": "custom/model", "provider_option": true}),
+ )
+ .await
+ .expect("streaming request")
+ .collect::<Vec<_>>()
+ .await
+ .into_iter()
+ .collect::<std::result::Result<Vec<_>, RequestError>>()
+ .expect("stream events")
+ });
+
+ assert_eq!(
+ events,
+ vec![
+ ChatCompletionStreamEvent::Data(json!({"chunk": 1})),
+ ChatCompletionStreamEvent::Done,
+ ]
+ );
+ assert_eq!(
+ captured_request
+ .lock()
+ .expect("captured request lock")
+ .as_ref(),
+ Some(&(
+ "https://example.com/v1/chat/completions".to_string(),
+ "Bearer secret".to_string(),
+ "enabled".to_string(),
+ json!({"model": "custom/model", "provider_option": true}).to_string(),
+ ))
+ );
+}
+
+#[test]
+fn streaming_transport_does_not_synthesize_done_at_eof() {
+ let client = FakeHttpClient::create(|_| async move {
+ Ok(Response::builder()
+ .status(200)
+ .body(AsyncBody::from("data: {\"chunk\":1}\n\n"))?)
+ });
+
+ let events = block_on(async {
+ stream_chat_completion(
+ client.as_ref(),
+ "Compatible Provider",
+ "https://example.com/v1",
+ "secret",
+ &CustomHeaders::default(),
+ &json!({"model": "custom/model"}),
+ )
+ .await
+ .expect("streaming request")
+ .collect::<Vec<_>>()
+ .await
+ .into_iter()
+ .collect::<std::result::Result<Vec<_>, RequestError>>()
+ .expect("stream events")
+ });
+
+ assert_eq!(
+ events,
+ vec![ChatCompletionStreamEvent::Data(json!({"chunk": 1}))]
+ );
+}
+
+#[test]
+fn transport_preserves_typed_send_and_deserialization_errors() {
+ let client = FakeHttpClient::create(|_| async move { Err(anyhow!("network unavailable")) });
+ let error = block_on(non_streaming_chat_completion(
+ client.as_ref(),
+ "Compatible Provider",
+ "https://example.com/v1",
+ "secret",
+ &CustomHeaders::default(),
+ &json!({"model": "custom/model"}),
+ ))
+ .expect_err("send error");
+ assert!(matches!(
+ error,
+ RequestError::HttpSend { provider, .. } if provider == "Compatible Provider"
+ ));
+
+ let client = FakeHttpClient::create(|_| async move {
+ Ok(Response::builder()
+ .status(200)
+ .body(AsyncBody::from("not JSON"))?)
+ });
+ let error = block_on(non_streaming_chat_completion(
+ client.as_ref(),
+ "Compatible Provider",
+ "https://example.com/v1",
+ "secret",
+ &CustomHeaders::default(),
+ &json!({"model": "custom/model"}),
+ ))
+ .expect_err("deserialization error");
+ assert!(matches!(
+ error,
+ RequestError::DeserializeResponse { provider, .. }
+ if provider == "Compatible Provider"
+ ));
+}
+
+#[test]
+fn non_streaming_transport_supports_custom_envelopes_and_provider_errors() {
+ let client = FakeHttpClient::create(|request| async move {
+ assert_eq!(request.headers()["x-compatible-provider"], "enabled");
+ Ok(Response::builder()
+ .status(200)
+ .body(AsyncBody::from(r#"{"result":"ok"}"#))?)
+ });
+ let extra_headers = CustomHeaders::new(vec![(
+ HeaderName::from_static("x-compatible-provider"),
+ HeaderValue::from_static("enabled"),
+ )]);
+ let response = block_on(non_streaming_chat_completion(
+ client.as_ref(),
+ "Compatible Provider",
+ "https://example.com/v1",
+ "secret",
+ &extra_headers,
+ &json!({"model": "custom/model", "custom_option": "value"}),
+ ))
+ .expect("non-streaming request");
+ assert_eq!(response, json!({"result": "ok"}));
+
+ let client = FakeHttpClient::create(|_| async move {
+ Ok(Response::builder()
+ .status(StatusCode::BAD_REQUEST)
+ .body(AsyncBody::from("invalid custom option"))?)
+ });
+ let error = block_on(non_streaming_chat_completion(
+ client.as_ref(),
+ "Compatible Provider",
+ "https://example.com/v1",
+ "secret",
+ &CustomHeaders::default(),
+ &json!({"model": "custom/model"}),
+ ))
+ .expect_err("provider error");
+ match error {
+ RequestError::HttpResponseError {
+ provider,
+ status_code,
+ body,
+ ..
+ } => {
+ assert_eq!(provider, "Compatible Provider");
+ assert_eq!(status_code, StatusCode::BAD_REQUEST);
+ assert_eq!(body, "invalid custom option");
+ }
+ error => panic!("unexpected transport error: {error}"),
+ }
+}
diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rs
index 12f99440e3e..d858fc419d4 100644
--- a/crates/open_ai/src/completion.rs
+++ b/crates/open_ai/src/completion.rs
@@ -73,6 +73,9 @@ pub fn into_open_ai(
let mut messages = Vec::new();
let mut current_reasoning: Option<String> = None;
for message in request.messages {
+ let reasoning_details = interleaved_reasoning
+ .then(|| message.reasoning_details.clone())
+ .flatten();
for content in message.content {
match content {
MessageContent::Thinking { text, .. } if interleaved_reasoning => {
@@ -91,6 +94,7 @@ pub fn into_open_ai(
MessagePart::Text { text },
message.role,
&mut messages,
+ reasoning_details.clone(),
);
if let Some(reasoning) = current_reasoning.take() {
if let Some(crate::RequestMessage::Assistant {
@@ -114,6 +118,7 @@ pub fn into_open_ai(
},
message.role,
&mut messages,
+ reasoning_details.clone(),
);
}
MessageContent::ToolUse(tool_use) => {
@@ -129,12 +134,19 @@ pub fn into_open_ai(
function: FunctionContent {
name: tool_use.name.to_string(),
arguments: serde_json::to_string(input).unwrap_or_default(),
+ thought_signature: interleaved_reasoning
+ .then(|| tool_use.thought_signature.clone())
+ .flatten(),
},
},
};
- if let Some(crate::RequestMessage::Assistant { tool_calls, .. }) =
- messages.last_mut()
+ if let Some(crate::RequestMessage::Assistant {
+ tool_calls,
+ reasoning_details: existing_reasoning_details,
+ ..
+ }) = messages.last_mut()
+ && existing_reasoning_details == &reasoning_details
{
tool_calls.push(tool_call);
} else {
@@ -142,6 +154,7 @@ pub fn into_open_ai(
content: None,
tool_calls: vec![tool_call],
reasoning_content: current_reasoning.take(),
+ reasoning_details: reasoning_details.clone(),
});
}
}
@@ -679,17 +692,21 @@ fn add_message_content_part(
new_part: MessagePart,
role: Role,
messages: &mut Vec<crate::RequestMessage>,
+ reasoning_details: Option<Arc<serde_json::Value>>,
) {
match (role, messages.last_mut()) {
(Role::User, Some(crate::RequestMessage::User { content }))
- | (
+ | (Role::System, Some(crate::RequestMessage::System { content, .. })) => {
+ content.push_part(new_part);
+ }
+ (
Role::Assistant,
Some(crate::RequestMessage::Assistant {
content: Some(content),
+ reasoning_details: existing_reasoning_details,
..
}),
- )
- | (Role::System, Some(crate::RequestMessage::System { content, .. })) => {
+ ) if existing_reasoning_details == &reasoning_details => {
content.push_part(new_part);
}
_ => {
@@ -701,6 +718,7 @@ fn add_message_content_part(
content: Some(crate::MessageContent::from(vec![new_part])),
tool_calls: Vec::new(),
reasoning_content: None,
+ reasoning_details,
},
Role::System => crate::RequestMessage::System {
content: crate::MessageContent::from(vec![new_part]),
@@ -710,14 +728,70 @@ fn add_message_content_part(
}
}
+/// Accumulates structured reasoning metadata from compatible providers.
+///
+/// Array entries are matched by `index` and then `id`. Fragmented `text`,
+/// `summary`, and `data` fields are concatenated while other non-null fields
+/// replace their previous values.
+///
+/// # Examples
+///
+/// ```
+/// use open_ai::completion::ReasoningDetailsAccumulator;
+/// use serde_json::json;
+///
+/// let mut accumulator = ReasoningDetailsAccumulator::default();
+/// accumulator.push(json!([{"index": 0, "text": "first "}]));
+/// let details = accumulator
+/// .push(json!([{"index": 0, "text": "second"}]))
+/// .expect("non-empty reasoning details");
+///
+/// assert_eq!(details[0]["text"], "first second");
+/// ```
+#[derive(Debug, Default)]
+pub struct ReasoningDetailsAccumulator {
+ accumulated: Option<serde_json::Value>,
+}
+
+impl ReasoningDetailsAccumulator {
+ /// Merges `chunk` and returns the updated metadata snapshot.
+ ///
+ /// `null` and empty arrays do not replace previously accumulated metadata
+ /// and return `None`.
+ pub fn push(&mut self, chunk: serde_json::Value) -> Option<serde_json::Value> {
+ match chunk {
+ serde_json::Value::Null => None,
+ serde_json::Value::Array(chunks) if chunks.is_empty() => None,
+ serde_json::Value::Array(chunks) => {
+ let mut details = match self.accumulated.take() {
+ Some(serde_json::Value::Array(details)) => details,
+ _ => Vec::new(),
+ };
+ for chunk in chunks {
+ merge_reasoning_detail(&mut details, chunk);
+ }
+ let accumulated = serde_json::Value::Array(details);
+ self.accumulated = Some(accumulated.clone());
+ Some(accumulated)
+ }
+ chunk => {
+ self.accumulated = Some(chunk.clone());
+ Some(chunk)
+ }
+ }
+ }
+}
+
pub struct OpenAiEventMapper {
tool_calls_by_index: HashMap<usize, RawToolCall>,
+ reasoning_details: ReasoningDetailsAccumulator,
}
impl OpenAiEventMapper {
pub fn new() -> Self {
Self {
tool_calls_by_index: HashMap::default(),
+ reasoning_details: ReasoningDetailsAccumulator::default(),
}
}
@@ -743,11 +817,21 @@ impl OpenAiEventMapper {
&& let Some(prompt_tokens) = usage.prompt_tokens
&& let Some(completion_tokens) = usage.completion_tokens
{
+ let cache_creation_input_tokens = usage
+ .prompt_tokens_details
+ .as_ref()
+ .map_or(0, |details| details.cache_write_tokens);
+ let cache_read_input_tokens = usage
+ .prompt_tokens_details
+ .as_ref()
+ .map_or(0, |details| details.cached_tokens);
events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
- input_tokens: prompt_tokens,
+ input_tokens: prompt_tokens
+ .saturating_sub(cache_creation_input_tokens)
+ .saturating_sub(cache_read_input_tokens),
output_tokens: completion_tokens,
- cache_creation_input_tokens: 0,
- cache_read_input_tokens: 0,
+ cache_creation_input_tokens,
+ cache_read_input_tokens,
})));
}
@@ -756,6 +840,13 @@ impl OpenAiEventMapper {
};
if let Some(delta) = choice.delta.as_ref() {
+ if let Some(reasoning_details) = delta.reasoning_details.clone()
+ && let Some(reasoning_details) = self.reasoning_details.push(reasoning_details)
+ {
+ events.push(Ok(LanguageModelCompletionEvent::ReasoningDetails(
+ reasoning_details,
+ )));
+ }
if let Some(reasoning) = delta.reasoning.clone() {
push_thinking_event(reasoning, &mut events);
}
@@ -788,6 +879,10 @@ impl OpenAiEventMapper {
if let Some(arguments) = function.arguments.clone() {
entry.arguments.push_str(&arguments);
}
+
+ if let Some(thought_signature) = function.thought_signature.clone() {
+ entry.thought_signature = Some(thought_signature);
+ }
}
if !entry.id.is_empty() && !entry.name.is_empty() {
@@ -801,7 +896,7 @@ impl OpenAiEventMapper {
is_input_complete: false,
input: LanguageModelToolUseInput::Json(input),
raw_input: entry.arguments.clone(),
- thought_signature: None,
+ thought_signature: entry.thought_signature.clone(),
},
)));
}
@@ -824,7 +919,7 @@ impl OpenAiEventMapper {
is_input_complete: true,
input: LanguageModelToolUseInput::Json(input),
raw_input: tool_call.arguments.clone(),
- thought_signature: None,
+ thought_signature: tool_call.thought_signature.clone(),
},
)),
Err(error) => Ok(LanguageModelCompletionEvent::ToolUseJsonParseError {
@@ -861,11 +956,49 @@ fn push_thinking_event(
}
}
+fn merge_reasoning_detail(details: &mut Vec<serde_json::Value>, chunk: serde_json::Value) {
+ let index = chunk.get("index").and_then(serde_json::Value::as_u64);
+ let target_index = index
+ .and_then(|index| {
+ details.iter().position(|detail| {
+ detail.get("index").and_then(serde_json::Value::as_u64) == Some(index)
+ })
+ })
+ .or_else(|| {
+ let id = chunk.get("id").and_then(serde_json::Value::as_str)?;
+ details
+ .iter()
+ .position(|detail| detail.get("id").and_then(serde_json::Value::as_str) == Some(id))
+ });
+ let Some(target_index) = target_index else {
+ details.push(chunk);
+ return;
+ };
+ let (Some(target), Some(chunk)) = (details[target_index].as_object_mut(), chunk.as_object())
+ else {
+ return;
+ };
+ for (key, value) in chunk {
+ if matches!(key.as_str(), "text" | "summary" | "data")
+ && let Some(fragment) = value.as_str()
+ && let Some(existing) = target.get(key).and_then(serde_json::Value::as_str)
+ {
+ target.insert(
+ key.clone(),
+ serde_json::Value::String(format!("{existing}{fragment}")),
+ );
+ } else if !value.is_null() {
+ target.insert(key.clone(), value.clone());
+ }
+ }
+}
+
#[derive(Default)]
struct RawToolCall {
id: String,
name: String,
arguments: String,
+ thought_signature: Option<String>,
}
pub struct OpenAiResponseEventMapper {
@@ -3930,7 +4063,7 @@ mod tests {
raw_input: tool_arguments.clone(),
input: LanguageModelToolUseInput::Json(tool_input),
is_input_complete: true,
- thought_signature: None,
+ thought_signature: Some("thought-signature".into()),
};
let tool_result = LanguageModelToolResult {
tool_use_id: tool_use_id,
@@ -3961,7 +4094,11 @@ mod tests {
MessageContent::ToolUse(tool_use),
],
cache: false,
- reasoning_details: None,
+ reasoning_details: Some(Arc::new(json!([{
+ "id": "reasoning-1",
+ "type": "reasoning.encrypted",
+ "data": "encrypted"
+ }]))),
},
LanguageModelRequestMessage {
role: Role::Assistant,
@@ -3998,8 +4135,13 @@ mod tests {
{
"role": "assistant",
"content": "Searching now.",
- "tool_calls": [{"id": "call-1", "type": "function", "function": {"name": "search", "arguments": tool_arguments}}],
- "reasoning_content": "I should search"
+ "tool_calls": [{"id": "call-1", "type": "function", "function": {"name": "search", "arguments": tool_arguments, "thought_signature": "thought-signature"}}],
+ "reasoning_content": "I should search",
+ "reasoning_details": [{
+ "id": "reasoning-1",
+ "type": "reasoning.encrypted",
+ "data": "encrypted"
+ }]
},
{"role": "tool", "content": "result", "tool_call_id": "call-1"}
])
@@ -4044,6 +4186,7 @@ mod tests {
reasoning: Some("thinking".into()),
tool_calls: None,
reasoning_content: None,
+ reasoning_details: None,
}),
finish_reason: None,
}],
@@ -4059,6 +4202,110 @@ mod tests {
);
}
+ #[test]
+ fn stream_merges_reasoning_details_and_maps_compatible_usage_and_signatures() {
+ let response_events = serde_json::from_value(json!([
+ {
+ "choices": [{
+ "index": 0,
+ "delta": {
+ "reasoning_details": [{
+ "id": "reasoning-1",
+ "index": 0,
+ "type": "reasoning.text",
+ "text": "first "
+ }],
+ "tool_calls": [{
+ "index": 0,
+ "id": "call-1",
+ "function": {
+ "name": "search",
+ "arguments": "{",
+ "thought_signature": "signature"
+ }
+ }]
+ },
+ "finish_reason": null
+ }],
+ "usage": null
+ },
+ {
+ "choices": [{
+ "index": 0,
+ "delta": {
+ "reasoning_details": [{
+ "id": "reasoning-1",
+ "index": 0,
+ "text": "second"
+ }],
+ "tool_calls": [{
+ "index": 0,
+ "function": {
+ "arguments": "}"
+ }
+ }]
+ },
+ "finish_reason": "tool_calls"
+ }],
+ "usage": null
+ },
+ {
+ "choices": [],
+ "usage": {
+ "prompt_tokens": 10000,
+ "completion_tokens": 500,
+ "total_tokens": 10500,
+ "prompt_tokens_details": {
+ "cached_tokens": 6000,
+ "cache_write_tokens": 1000
+ }
+ }
+ }
+ ]))
+ .expect("valid compatible Chat Completions events");
+ let events = map_completion_events(response_events);
+
+ assert!(events.iter().any(|event| {
+ matches!(
+ event,
+ LanguageModelCompletionEvent::ReasoningDetails(details)
+ if details[0]["text"] == "first second"
+ )
+ }));
+ assert!(events.iter().any(|event| {
+ matches!(
+ event,
+ LanguageModelCompletionEvent::ToolUse(tool_use)
+ if tool_use.is_input_complete
+ && tool_use.thought_signature.as_deref() == Some("signature")
+ )
+ }));
+ assert!(events.iter().any(|event| {
+ matches!(
+ event,
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
+ input_tokens: 3_000,
+ output_tokens: 500,
+ cache_creation_input_tokens: 1_000,
+ cache_read_input_tokens: 6_000,
+ })
+ )
+ }));
+ }
+
+ #[test]
+ fn reasoning_details_accumulator_replaces_an_incompatible_previous_shape() {
+ let mut accumulator = ReasoningDetailsAccumulator::default();
+ assert_eq!(
+ accumulator.push(json!({"summary": "provider-defined"})),
+ Some(json!({"summary": "provider-defined"}))
+ );
+ assert_eq!(
+ accumulator.push(json!([{"index": 0, "text": "reasoning"}])),
+ Some(json!([{"index": 0, "text": "reasoning"}]))
+ );
+ }
+
#[test]
fn stream_maps_preserves_tool_id_and_name_across_empty_deltas() {
// DashScope sends id="" and name="" in subsequent tool_calls delta
@@ -4080,9 +4327,11 @@ mod tests {
function: Some(FunctionChunk {
name: Some("list_directory".into()),
arguments: Some("".into()),
+ thought_signature: None,
}),
}]),
reasoning_content: None,
+ reasoning_details: None,
}),
finish_reason: None,
}],
@@ -4102,9 +4351,11 @@ mod tests {
function: Some(FunctionChunk {
name: Some("".into()),
arguments: Some("{\"path\": \"".into()),
+ thought_signature: None,
}),
}]),
reasoning_content: None,
+ reasoning_details: None,
}),
finish_reason: None,
}],
@@ -4123,9 +4374,11 @@ mod tests {
function: Some(FunctionChunk {
name: Some("".into()),
arguments: Some("blog-scraper\"}".into()),
+ thought_signature: None,
}),
}]),
reasoning_content: None,
+ reasoning_details: None,
}),
finish_reason: None,
}],
diff --git a/crates/open_ai/src/open_ai.rs b/crates/open_ai/src/open_ai.rs
index 093c58649ef..8bc4af65f75 100644
--- a/crates/open_ai/src/open_ai.rs
+++ b/crates/open_ai/src/open_ai.rs
@@ -2,6 +2,9 @@ pub mod batches;
pub mod completion;
pub mod responses;
+#[cfg(test)]
+mod chat_completion_transport_tests;
+
use anyhow::{Context as _, Result, anyhow};
use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
use http_client::{
@@ -12,7 +15,7 @@ use http_client::{
pub use language_model_core::ReasoningEffort;
use serde::{Deserialize, Serialize};
use serde_json::Value;
-use std::{convert::TryFrom, future::Future};
+use std::{convert::TryFrom, future::Future, io};
use strum::EnumIter;
use thiserror::Error;
@@ -605,6 +608,9 @@ pub enum RequestMessage {
tool_calls: Vec<ToolCall>,
#[serde(default, skip_serializing_if = "Option::is_none")]
reasoning_content: Option<String>,
+ /// Provider-defined reasoning metadata required for replay.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ reasoning_details: Option<std::sync::Arc<Value>>,
},
User {
content: MessageContent,
@@ -688,6 +694,9 @@ pub enum ToolCallContent {
pub struct FunctionContent {
pub name: String,
pub arguments: String,
+ /// Provider-defined metadata required to replay a reasoning tool call.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub thought_signature: Option<String>,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
@@ -716,6 +725,9 @@ pub struct ResponseMessageDelta {
pub tool_calls: Option<Vec<ToolCallChunk>>,
#[serde(default, skip_serializing_if = "is_none_or_empty")]
pub reasoning_content: Option<String>,
+ /// Provider-defined structured reasoning metadata.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub reasoning_details: Option<Value>,
}
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
@@ -733,6 +745,20 @@ pub struct ToolCallChunk {
pub struct FunctionChunk {
pub name: Option<String>,
pub arguments: Option<String>,
+ /// Provider-defined metadata required to replay a reasoning tool call.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub thought_signature: Option<String>,
+}
+
+/// Reports prompt-cache token usage from compatible providers.
+#[derive(Clone, Serialize, Deserialize, Debug, Default)]
+pub struct PromptTokensDetails {
+ /// Tokens read from a prompt cache.
+ #[serde(default)]
+ pub cached_tokens: u64,
+ /// Tokens written to a prompt cache.
+ #[serde(default)]
+ pub cache_write_tokens: u64,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
@@ -740,6 +766,9 @@ pub struct Usage {
pub prompt_tokens: Option<u64>,
pub completion_tokens: Option<u64>,
pub total_tokens: Option<u64>,
+ /// Prompt-cache usage when reported by the provider.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub prompt_tokens_details: Option<PromptTokensDetails>,
}
#[derive(Serialize, Deserialize, Debug)]
@@ -749,6 +778,10 @@ pub struct ChoiceDelta {
pub finish_reason: Option<String>,
}
+/// An error produced while sending an OpenAI-compatible request.
+///
+/// Transport and wire-format failures retain their category so callers can
+/// present an appropriate error or translate it into provider-specific errors.
#[derive(Error, Debug)]
pub enum RequestError {
#[error("HTTP response error from {provider}'s API: status {status_code} - {body:?}")]
@@ -756,7 +789,37 @@ pub enum RequestError {
provider: String,
status_code: StatusCode,
body: String,
- headers: HeaderMap<HeaderValue>,
+ headers: Box<HeaderMap<HeaderValue>>,
+ },
+ #[error("error serializing request to {provider}'s API")]
+ SerializeRequest {
+ provider: String,
+ #[source]
+ error: serde_json::Error,
+ },
+ #[error("error building request body for {provider}'s API")]
+ BuildRequestBody {
+ provider: String,
+ #[source]
+ error: http_client::http::Error,
+ },
+ #[error("error sending HTTP request to {provider}'s API")]
+ HttpSend {
+ provider: String,
+ #[source]
+ error: anyhow::Error,
+ },
+ #[error("I/O error reading response from {provider}'s API")]
+ ReadResponse {
+ provider: String,
+ #[source]
+ error: io::Error,
+ },
+ #[error("error deserializing {provider}'s API response")]
+ DeserializeResponse {
+ provider: String,
+ #[source]
+ error: serde_json::Error,
},
#[error(transparent)]
Other(#[from] anyhow::Error),
@@ -780,50 +843,162 @@ pub struct ResponseStreamEvent {
pub usage: Option<Usage>,
}
+/// A framed Chat Completions server-sent event.
+///
+/// `Done` is distinct from the underlying response body ending so callers can
+/// tell whether the server completed the stream according to the protocol.
+#[derive(Debug, PartialEq)]
+pub enum ChatCompletionStreamEvent {
+ /// A JSON payload from a `data` field.
+ Data(Value),
+ /// The protocol terminator `data: [DONE]`.
+ Done,
+}
+
+/// Sends a non-streaming compatible Chat Completions request.
+///
+/// The request may use any serializable envelope. The response body is returned
+/// as JSON so the caller can deserialize provider-specific extensions.
+///
+/// # Errors
+///
+/// Returns [`RequestError`] when request serialization or construction fails,
+/// the HTTP request fails, the server returns a non-success status, the response
+/// body cannot be read, or the response is not valid JSON.
+pub async fn non_streaming_chat_completion<RequestBody>(
+ client: &dyn HttpClient,
+ provider_name: &str,
+ api_url: &str,
+ api_key: &str,
+ extra_headers: &CustomHeaders,
+ request: &RequestBody,
+) -> Result<Value, RequestError>
+where
+ RequestBody: Serialize + ?Sized,
+{
+ let request = chat_completion_request(provider_name, api_url, api_key, extra_headers, request)?;
+ let mut response = client
+ .send(request)
+ .await
+ .map_err(|error| RequestError::HttpSend {
+ provider: provider_name.to_string(),
+ error,
+ })?;
+ if !response.status().is_success() {
+ return Err(http_response_error(provider_name, &mut response).await);
+ }
+
+ let body =
+ read_response_body(&mut response)
+ .await
+ .map_err(|error| RequestError::ReadResponse {
+ provider: provider_name.to_string(),
+ error,
+ })?;
+ serde_json::from_str(&body).map_err(|error| RequestError::DeserializeResponse {
+ provider: provider_name.to_string(),
+ error,
+ })
+}
+
pub async fn non_streaming_completion(
client: &dyn HttpClient,
api_url: &str,
api_key: &str,
request: Request,
) -> Result<Response, RequestError> {
- let uri = format!("{api_url}/chat/completions");
- let request_builder = HttpRequest::builder()
- .method(Method::POST)
- .uri(uri)
- .header("Content-Type", "application/json")
- .header("Authorization", format!("Bearer {}", api_key.trim()));
-
- let request = request_builder
- .body(AsyncBody::from(
- serde_json::to_string(&request).map_err(|e| RequestError::Other(e.into()))?,
- ))
- .map_err(|e| RequestError::Other(e.into()))?;
+ let response = non_streaming_chat_completion(
+ client,
+ "openai",
+ api_url,
+ api_key,
+ &CustomHeaders::default(),
+ &request,
+ )
+ .await?;
+ serde_json::from_value(response).map_err(|error| RequestError::Other(error.into()))
+}
- let mut response = client.send(request).await?;
- if response.status().is_success() {
- let mut body = String::new();
- response
- .body_mut()
- .read_to_string(&mut body)
- .await
- .map_err(|e| RequestError::Other(e.into()))?;
+/// Starts a streaming compatible Chat Completions request.
+///
+/// The returned stream preserves the distinction between a `[DONE]` event and
+/// the response body ending without a protocol terminator.
+///
+/// # Errors
+///
+/// Returns [`RequestError`] before streaming begins when request serialization
+/// or construction fails, the HTTP request fails, or the server returns a
+/// non-success status. The returned stream yields [`RequestError`] when the
+/// response body cannot be read or a `data` field is not valid JSON.
+pub async fn stream_chat_completion<RequestBody>(
+ client: &dyn HttpClient,
+ provider_name: &str,
+ api_url: &str,
+ api_key: &str,
+ extra_headers: &CustomHeaders,
+ request: &RequestBody,
+) -> Result<
+ BoxStream<'static, std::result::Result<ChatCompletionStreamEvent, RequestError>>,
+ RequestError,
+>
+where
+ RequestBody: Serialize + ?Sized,
+{
+ let request = chat_completion_request(provider_name, api_url, api_key, extra_headers, request)?;
+ let mut response = client
+ .send(request)
+ .await
+ .map_err(|error| RequestError::HttpSend {
+ provider: provider_name.to_string(),
+ error,
+ })?;
+ if !response.status().is_success() {
+ return Err(http_response_error(provider_name, &mut response).await);
+ }
- serde_json::from_str(&body).map_err(|e| RequestError::Other(e.into()))
- } else {
- let mut body = String::new();
- response
- .body_mut()
- .read_to_string(&mut body)
- .await
- .map_err(|e| RequestError::Other(e.into()))?;
+ let lines = BufReader::new(response.into_body()).lines();
+ let provider_name = provider_name.to_string();
+ Ok(futures::stream::try_unfold(
+ (lines, false, provider_name),
+ async move |(mut lines, is_done, provider_name)| {
+ if is_done {
+ return Ok(None);
+ }
- Err(RequestError::HttpResponseError {
- provider: "openai".to_owned(),
- status_code: response.status(),
- body,
- headers: response.headers().clone(),
- })
- }
+ loop {
+ let Some(line) = lines.next().await else {
+ return Ok(None);
+ };
+ let line = line.map_err(|error| RequestError::ReadResponse {
+ provider: provider_name.clone(),
+ error,
+ })?;
+ let Some(data) = line
+ .strip_prefix("data: ")
+ .or_else(|| line.strip_prefix("data:"))
+ else {
+ continue;
+ };
+ if data == "[DONE]" {
+ return Ok(Some((
+ ChatCompletionStreamEvent::Done,
+ (lines, true, provider_name),
+ )));
+ }
+ let value = serde_json::from_str(data).map_err(|error| {
+ RequestError::DeserializeResponse {
+ provider: provider_name.clone(),
+ error,
+ }
+ })?;
+ return Ok(Some((
+ ChatCompletionStreamEvent::Data(value),
+ (lines, false, provider_name),
+ )));
+ }
+ },
+ )
+ .boxed())
}
pub async fn stream_completion(
@@ -834,68 +1009,93 @@ pub async fn stream_completion(
request: Request,
extra_headers: &CustomHeaders,
) -> Result<BoxStream<'static, Result<ResponseStreamEvent>>, RequestError> {
- let uri = format!("{api_url}/chat/completions");
- let request = HttpRequest::builder()
+ let events = stream_chat_completion(
+ client,
+ provider_name,
+ api_url,
+ api_key,
+ extra_headers,
+ &request,
+ )
+ .await?;
+ Ok(events
+ .filter_map(|event| async move {
+ let value = match event {
+ Ok(ChatCompletionStreamEvent::Data(value)) => value,
+ Ok(ChatCompletionStreamEvent::Done) => return None,
+ Err(error) => return Some(Err(anyhow!(error))),
+ };
+ match ResponseStreamResult::deserialize(&value) {
+ Ok(ResponseStreamResult::Ok(response)) => Some(Ok(response)),
+ Ok(ResponseStreamResult::Err { error }) => Some(Err(anyhow!(error.message))),
+ Err(error) => {
+ log::error!(
+ "Failed to parse OpenAI response into ResponseStreamResult: `{}`\n\
+ Response: `{}`",
+ error,
+ value,
+ );
+ Some(Err(anyhow!(error)))
+ }
+ }
+ })
+ .boxed())
+}
+
+fn chat_completion_request<RequestBody>(
+ provider_name: &str,
+ api_url: &str,
+ api_key: &str,
+ extra_headers: &CustomHeaders,
+ request: &RequestBody,
+) -> std::result::Result<HttpRequest<AsyncBody>, RequestError>
+where
+ RequestBody: Serialize + ?Sized,
+{
+ let body = serde_json::to_string(request).map_err(|error| RequestError::SerializeRequest {
+ provider: provider_name.to_string(),
+ error,
+ })?;
+ HttpRequest::builder()
.method(Method::POST)
- .uri(uri)
+ .uri(format!("{api_url}/chat/completions"))
.header("Content-Type", "application/json")
.header("Authorization", format!("Bearer {}", api_key.trim()))
.extra_headers(extra_headers)
- .body(AsyncBody::from(
- serde_json::to_string(&request).map_err(|e| RequestError::Other(e.into()))?,
- ))
- .map_err(|e| RequestError::Other(e.into()))?;
-
- let mut response = client.send(request).await?;
- if response.status().is_success() {
- let reader = BufReader::new(response.into_body());
- Ok(reader
- .lines()
- .filter_map(|line| async move {
- match line {
- Ok(line) => {
- let line = line.strip_prefix("data: ").or_else(|| line.strip_prefix("data:"))?;
- if line == "[DONE]" {
- None
- } else {
- match serde_json::from_str(line) {
- Ok(ResponseStreamResult::Ok(response)) => Some(Ok(response)),
- Ok(ResponseStreamResult::Err { error }) => {
- Some(Err(anyhow!(error.message)))
- }
- Err(error) => {
- log::error!(
- "Failed to parse OpenAI response into ResponseStreamResult: `{}`\n\
- Response: `{}`",
- error,
- line,
- );
- Some(Err(anyhow!(error)))
- }
- }
- }
- }
- Err(error) => Some(Err(anyhow!(error))),
- }
- })
- .boxed())
- } else {
- let mut body = String::new();
- response
- .body_mut()
- .read_to_string(&mut body)
- .await
- .map_err(|e| RequestError::Other(e.into()))?;
-
- Err(RequestError::HttpResponseError {
- provider: provider_name.to_owned(),
- status_code: response.status(),
- body,
- headers: response.headers().clone(),
+ .body(AsyncBody::from(body))
+ .map_err(|error| RequestError::BuildRequestBody {
+ provider: provider_name.to_string(),
+ error,
})
+}
+
+async fn http_response_error(
+ provider_name: &str,
+ response: &mut http_client::Response<AsyncBody>,
+) -> RequestError {
+ let body = match read_response_body(response).await {
+ Ok(body) => body,
+ Err(error) => {
+ return RequestError::ReadResponse {
+ provider: provider_name.to_string(),
+ error,
+ };
+ }
+ };
+ RequestError::HttpResponseError {
+ provider: provider_name.to_string(),
+ status_code: response.status(),
+ body,
+ headers: Box::new(response.headers().clone()),
}
}
+async fn read_response_body(response: &mut http_client::Response<AsyncBody>) -> io::Result<String> {
+ let mut body = String::new();
+ response.body_mut().read_to_string(&mut body).await?;
+ Ok(body)
+}
+
#[derive(Copy, Clone, Serialize, Deserialize)]
pub enum OpenAiEmbeddingModel {
#[serde(rename = "text-embedding-3-small")]
@@ -977,6 +1177,26 @@ impl From<RequestError> for language_model_core::LanguageModelCompletionError {
Self::from_http_status(provider.into(), status_code, body, retry_after)
}
+ RequestError::SerializeRequest { provider, error } => Self::SerializeRequest {
+ provider: provider.into(),
+ error,
+ },
+ RequestError::BuildRequestBody { provider, error } => Self::BuildRequestBody {
+ provider: provider.into(),
+ error,
+ },
+ RequestError::HttpSend { provider, error } => Self::HttpSend {
+ provider: provider.into(),
+ error,
+ },
+ RequestError::ReadResponse { provider, error } => Self::ApiReadResponseError {
+ provider: provider.into(),
+ error,
+ },
+ RequestError::DeserializeResponse { provider, error } => Self::DeserializeResponse {
+ provider: provider.into(),
+ error,
+ },
RequestError::Other(e) => Self::Other(e),
}
}
diff --git a/crates/open_ai/src/responses.rs b/crates/open_ai/src/responses.rs
index ac9c38c6d8d..6d6c903ea38 100644
--- a/crates/open_ai/src/responses.rs
+++ b/crates/open_ai/src/responses.rs
@@ -758,7 +758,7 @@ pub async fn compact_response(
provider: provider_name.to_owned(),
status_code: response.status(),
body,
- headers: response.headers().clone(),
+ headers: Box::new(response.headers().clone()),
})
}
}
@@ -948,7 +948,7 @@ pub async fn stream_response(
provider: provider_name.to_owned(),
status_code: response.status(),
body,
- headers: response.headers().clone(),
+ headers: Box::new(response.headers().clone()),
})
}
}
diff --git a/crates/open_router/Cargo.toml b/crates/open_router/Cargo.toml
index 2cc5d3d00e2..84b08e3cf68 100644
--- a/crates/open_router/Cargo.toml
+++ b/crates/open_router/Cargo.toml
@@ -20,9 +20,13 @@ anyhow.workspace = true
futures.workspace = true
http_client.workspace = true
language_model_core.workspace = true
+open_ai.workspace = true
schemars = { workspace = true, optional = true }
serde.workspace = true
serde_json.workspace = true
settings.workspace = true
strum.workspace = true
thiserror.workspace = true
+
+[dev-dependencies]
+http_client = { workspace = true, features = ["test-support"] }
diff --git a/crates/open_router/src/open_router.rs b/crates/open_router/src/open_router.rs
index 006b1ffc97d..b2e6cc41c2d 100644
--- a/crates/open_router/src/open_router.rs
+++ b/crates/open_router/src/open_router.rs
@@ -1,8 +1,9 @@
use anyhow::{Result, anyhow};
-use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
+use futures::{AsyncReadExt, StreamExt, stream::BoxStream};
use http_client::{
AsyncBody, CustomHeaders, HttpClient, Method, Request as HttpRequest, RequestBuilderExt, http,
};
+use open_ai::ChatCompletionStreamEvent;
use serde::{Deserialize, Serialize};
use serde_json::Value;
pub use settings::DataCollection;
@@ -14,6 +15,7 @@ use strum::EnumString;
use thiserror::Error;
pub const OPEN_ROUTER_API_URL: &str = "https://openrouter.ai/api/v1";
+const OPEN_ROUTER_APP_TITLE: &str = "Zed";
fn extract_retry_after(headers: &http::HeaderMap) -> Option<std::time::Duration> {
if let Some(reset) = headers.get("X-RateLimit-Reset") {
@@ -439,6 +441,13 @@ pub struct ResponseStreamEvent {
pub usage: Option<Usage>,
}
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum ResponseStreamResult {
+ Response(ResponseStreamEvent),
+ Error(OpenRouterErrorResponse),
+}
+
#[derive(Serialize, Deserialize, Debug)]
pub struct Response {
pub id: String,
@@ -488,91 +497,50 @@ pub async fn stream_completion(
request: Request,
extra_headers: &CustomHeaders,
) -> Result<BoxStream<'static, Result<ResponseStreamEvent, OpenRouterError>>, OpenRouterError> {
- let uri = format!("{api_url}/chat/completions");
- let request = HttpRequest::builder()
- .method(Method::POST)
- .uri(uri)
- .header("Content-Type", "application/json")
- .header("Authorization", format!("Bearer {}", api_key))
- .header("HTTP-Referer", "https://zed.dev")
- .header("X-Title", "Zed Editor")
- .extra_headers(extra_headers)
- .body(AsyncBody::from(
- serde_json::to_string(&request).map_err(OpenRouterError::SerializeRequest)?,
- ))
- .map_err(OpenRouterError::BuildRequestBody)?;
- let mut response = client
- .send(request)
- .await
- .map_err(OpenRouterError::HttpSend)?;
-
- if response.status().is_success() {
- let reader = BufReader::new(response.into_body());
- Ok(reader
- .lines()
- .filter_map(|line| async move {
- match line {
- Ok(line) => {
- if line.starts_with(':') {
- return None;
- }
-
- let line = line.strip_prefix("data: ")?;
- if line == "[DONE]" {
- None
- } else {
- match serde_json::from_str::<ResponseStreamEvent>(line) {
- Ok(response) => Some(Ok(response)),
- Err(error) => {
- if line.trim().is_empty() {
- None
- } else {
- Some(Err(OpenRouterError::DeserializeResponse(error)))
- }
- }
- }
- }
- }
- Err(error) => Some(Err(OpenRouterError::ReadResponse(error))),
- }
- })
- .boxed())
- } else {
- let code = ApiErrorCode::from_status(response.status().as_u16());
-
- let mut body = String::new();
- response
- .body_mut()
- .read_to_string(&mut body)
+ let headers = completion_headers(extra_headers);
+ let events =
+ open_ai::stream_chat_completion(client, "OpenRouter", api_url, api_key, &headers, &request)
.await
- .map_err(OpenRouterError::ReadResponse)?;
-
- let error_response = match serde_json::from_str::<OpenRouterErrorResponse>(&body) {
- Ok(OpenRouterErrorResponse { error }) => error,
- Err(_) => OpenRouterErrorBody {
- code: response.status().as_u16(),
- message: body,
- metadata: None,
- },
- };
-
- match code {
- ApiErrorCode::RateLimitError => {
- let retry_after = extract_retry_after(response.headers());
- Err(OpenRouterError::RateLimit {
- retry_after: retry_after.unwrap_or_else(|| std::time::Duration::from_secs(60)),
- })
- }
- ApiErrorCode::OverloadedError => {
- let retry_after = extract_retry_after(response.headers());
- Err(OpenRouterError::ServerOverloaded { retry_after })
+ .map_err(OpenRouterError::from_chat_completion_request_error)?;
+ Ok(events
+ .filter_map(|event| async move {
+ let value = match event {
+ Ok(ChatCompletionStreamEvent::Data(value)) => value,
+ Ok(ChatCompletionStreamEvent::Done) => return None,
+ Err(error) => {
+ return Some(Err(OpenRouterError::ChatCompletion(error)));
+ }
+ };
+ match serde_json::from_value(value) {
+ Ok(ResponseStreamResult::Response(response)) => Some(Ok(response)),
+ Ok(ResponseStreamResult::Error(OpenRouterErrorResponse { error })) => {
+ Some(Err(OpenRouterError::ApiError(ApiError {
+ code: ApiErrorCode::from_status(error.code),
+ message: error.message,
+ })))
+ }
+ Err(error) => Some(Err(OpenRouterError::DeserializeResponse(error))),
}
- _ => Err(OpenRouterError::ApiError(ApiError {
- code: code,
- message: error_response.message,
- })),
- }
- }
+ })
+ .boxed())
+}
+
+fn completion_headers(extra_headers: &CustomHeaders) -> CustomHeaders {
+ let mut headers = Vec::with_capacity(extra_headers.iter().len() + 2);
+ headers.push((
+ http::HeaderName::from_static("http-referer"),
+ http::HeaderValue::from_static("https://zed.dev"),
+ ));
+ headers.push((
+ http::HeaderName::from_static("x-title"),
+ http::HeaderValue::from_static(OPEN_ROUTER_APP_TITLE),
+ ));
+ headers.extend(
+ extra_headers
+ .iter()
+ .map(|(name, value)| (name.clone(), value.clone())),
+ );
+ CustomHeaders::new(headers)
}
pub async fn list_models(
@@ -588,7 +556,7 @@ pub async fn list_models(
.header("Accept", "application/json")
.header("Authorization", format!("Bearer {}", api_key))
.header("HTTP-Referer", "https://zed.dev")
- .header("X-Title", "Zed Editor")
+ .header("X-Title", OPEN_ROUTER_APP_TITLE)
.extra_headers(extra_headers)
.body(AsyncBody::default())
.map_err(OpenRouterError::BuildRequestBody)?;
@@ -684,9 +652,6 @@ pub async fn list_models(
#[derive(Debug)]
pub enum OpenRouterError {
- /// Failed to serialize the HTTP request body to JSON
- SerializeRequest(serde_json::Error),
-
/// Failed to construct the HTTP request body
BuildRequestBody(http::Error),
@@ -707,6 +672,46 @@ pub enum OpenRouterError {
/// API returned an error response
ApiError(ApiError),
+
+ /// The shared Chat Completions transport failed.
+ ChatCompletion(open_ai::RequestError),
+}
+
+impl OpenRouterError {
+ fn from_chat_completion_request_error(error: open_ai::RequestError) -> Self {
+ let open_ai::RequestError::HttpResponseError {
+ status_code,
+ body,
+ headers,
+ ..
+ } = error
+ else {
+ return Self::ChatCompletion(error);
+ };
+ let code = ApiErrorCode::from_status(status_code.as_u16());
+ let error_response = match serde_json::from_str::<OpenRouterErrorResponse>(&body) {
+ Ok(OpenRouterErrorResponse { error }) => error,
+ Err(_) => OpenRouterErrorBody {
+ code: status_code.as_u16(),
+ message: body,
+ metadata: None,
+ },
+ };
+
+ match code {
+ ApiErrorCode::RateLimitError => Self::RateLimit {
+ retry_after: extract_retry_after(&headers)
+ .unwrap_or_else(|| std::time::Duration::from_secs(60)),
+ },
+ ApiErrorCode::OverloadedError => Self::ServerOverloaded {
+ retry_after: extract_retry_after(&headers),
+ },
+ _ => Self::ApiError(ApiError {
+ code,
+ message: error_response.message,
+ }),
+ }
+ }
}
#[derive(Debug, Serialize, Deserialize)]
@@ -790,7 +795,6 @@ impl From<OpenRouterError> for language_model_core::LanguageModelCompletionError
fn from(error: OpenRouterError) -> Self {
let provider = language_model_core::LanguageModelProviderName::new("OpenRouter");
match error {
- OpenRouterError::SerializeRequest(error) => Self::SerializeRequest { provider, error },
OpenRouterError::BuildRequestBody(error) => Self::BuildRequestBody { provider, error },
OpenRouterError::HttpSend(error) => Self::HttpSend { provider, error },
OpenRouterError::DeserializeResponse(error) => {
@@ -806,6 +810,7 @@ impl From<OpenRouterError> for language_model_core::LanguageModelCompletionError
retry_after,
},
OpenRouterError::ApiError(api_error) => api_error.into(),
+ OpenRouterError::ChatCompletion(error) => error.into(),
}
}
}
@@ -851,3 +856,96 @@ impl From<ApiError> for language_model_core::LanguageModelCompletionError {
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use futures::executor::block_on;
+ use http_client::{
+ FakeHttpClient, Response,
+ http::{HeaderName, HeaderValue},
+ };
+ use std::sync::{Arc, Mutex};
+
+ #[test]
+ fn completion_uses_shared_transport_with_open_router_headers() {
+ let captured_headers = Arc::new(Mutex::new(None));
+ let captured_headers_for_handler = captured_headers.clone();
+ let client = FakeHttpClient::create(move |request| {
+ let captured_headers = captured_headers_for_handler.clone();
+ async move {
+ captured_headers
+ .lock()
+ .expect("captured headers lock")
+ .replace(request.headers().clone());
+ Ok(Response::builder()
+ .status(200)
+ .body(AsyncBody::from(concat!(
+ "data: {\"id\":\"response-1\",\"created\":1,\"model\":\"vendor/model\",\"choices\":[],\"usage\":null}\n\n",
+ "data: [DONE]\n\n"
+ )))?)
+ }
+ });
+ let extra_headers = CustomHeaders::new(vec![(
+ HeaderName::from_static("x-custom-header"),
+ HeaderValue::from_static("custom-value"),
+ )]);
+ let request = Request {
+ model: "vendor/model".to_string(),
+ messages: vec![RequestMessage::User {
+ content: MessageContent::Plain("Hello".to_string()),
+ }],
+ stream: true,
+ session_id: None,
+ max_tokens: None,
+ stop: Vec::new(),
+ temperature: 0.4,
+ tool_choice: None,
+ parallel_tool_calls: None,
+ tools: Vec::new(),
+ reasoning: None,
+ usage: RequestUsage { include: true },
+ provider: None,
+ };
+
+ let responses = block_on(async {
+ stream_completion(
+ client.as_ref(),
+ OPEN_ROUTER_API_URL,
+ "secret",
+ request,
+ &extra_headers,
+ )
+ .await
+ .expect("streaming request")
+ .collect::<Vec<_>>()
+ .await
+ .into_iter()
+ .collect::<std::result::Result<Vec<_>, _>>()
+ .expect("stream responses")
+ });
+
+ assert_eq!(responses.len(), 1);
+ assert_eq!(responses[0].model, "vendor/model");
+ let headers = captured_headers.lock().expect("captured headers lock");
+ let headers = headers.as_ref().expect("captured headers");
+ assert_eq!(headers["http-referer"], "https://zed.dev");
+ assert_eq!(headers["x-title"], OPEN_ROUTER_APP_TITLE);
+ assert_eq!(headers["x-custom-header"], "custom-value");
+ }
+
+ #[test]
+ fn shared_transport_errors_retain_language_model_classification() {
+ let error = OpenRouterError::ChatCompletion(open_ai::RequestError::HttpSend {
+ provider: "OpenRouter".to_string(),
+ error: anyhow!("network unavailable"),
+ });
+ let error = language_model_core::LanguageModelCompletionError::from(error);
+
+ assert!(matches!(
+ error,
+ language_model_core::LanguageModelCompletionError::HttpSend { provider, .. }
+ if provider.0 == "OpenRouter"
+ ));
+ }
+}