Skip to content

Use Claude Code with your ChatGPT, Kimi, Cursor or Grok subscription via a local Anthropic-compatible proxy

Rust

fix OpenAI compatibility protocol edge cases

Raine Virta9b957bcd4e9e
diff --git a/src/openai_compat/mod.rs b/src/openai_compat/mod.rs
index 84a7263..591b62a 100644
--- a/src/openai_compat/mod.rs
+++ b/src/openai_compat/mod.rs
@@ -52,6 +52,17 @@ impl OpenAiError {
}
}
+ pub fn upstream_protocol(message: impl Into<String>) -> Self {
+ Self {
+ status: StatusCode::BAD_GATEWAY,
+ kind: "api_error".into(),
+ message: message.into().into_boxed_str(),
+ param: None,
+ code: Some("upstream_protocol_error".into()),
+ retry_after: None,
+ }
+ }
+
pub fn unsupported(param: impl Into<String>) -> Self {
let param = param.into();
Self {
diff --git a/src/openai_compat/request.rs b/src/openai_compat/request.rs
index 3d9e103..2d97eb9 100644
--- a/src/openai_compat/request.rs
+++ b/src/openai_compat/request.rs
@@ -114,11 +114,12 @@ pub fn parse_request(
}
validate_single_choice(object)?;
validate_parallel_tools(object)?;
+ validate_store(surface, object)?;
let max_tokens = parse_max_tokens(surface, object)?;
let tools = parse_tools(object.get("tools"), surface)?;
let tool_choice = parse_tool_choice(object.get("tool_choice"), &tools, surface)?;
let effort = parse_effort(surface, object)?;
- validate_cursor(provider, session_id, &messages, &tools)?;
+ validate_cursor(provider, session_id, stream, &messages, &tools)?;
let mut extra = Map::new();
if !system.is_empty() {
@@ -181,6 +182,20 @@ fn optional_bool(
}
}
+fn validate_store(surface: OpenAiSurface, object: &Map<String, Value>) -> Result<(), OpenAiError> {
+ if surface != OpenAiSurface::Responses {
+ return Ok(());
+ }
+ match object.get("store") {
+ None | Some(Value::Null | Value::Bool(false)) => Ok(()),
+ Some(Value::Bool(true)) => Err(OpenAiError::unsupported("store")),
+ Some(_) => Err(OpenAiError::invalid(
+ "'store' must be a boolean",
+ Some("store"),
+ )),
+ }
+}
+
fn parse_stream_options(value: Option<&Value>, stream: bool) -> Result<bool, OpenAiError> {
let Some(value) = value.filter(|value| !value.is_null()) else {
return Ok(false);
@@ -282,7 +297,10 @@ fn parse_effort(
let effort = value.as_str().ok_or_else(|| {
OpenAiError::invalid(
"Reasoning effort must be a string",
- Some("reasoning_effort"),
+ Some(match surface {
+ OpenAiSurface::ChatCompletions => "reasoning_effort",
+ OpenAiSurface::Responses => "reasoning.effort",
+ }),
)
})?;
match effort {
@@ -366,7 +384,7 @@ fn parse_chat_messages(value: Option<&Value>) -> Result<(Vec<Message>, Vec<Strin
}
"tool" => {
let id = required_string(object, "tool_call_id", &format!("{param}.tool_call_id"))?;
- if !calls.contains(&id) {
+ if !calls.remove(&id) {
return Err(OpenAiError::invalid(
format!("Tool result references unknown call '{id}'"),
Some(format!("{param}.tool_call_id")),
@@ -502,7 +520,7 @@ fn parse_responses_input(
&param,
)?;
let id = required_string(object, "call_id", &format!("{param}.call_id"))?;
- if !calls.contains(&id) {
+ if !calls.remove(&id) {
return Err(OpenAiError::invalid(
format!("Function output references unknown call '{id}'"),
Some(format!("{param}.call_id")),
@@ -789,7 +807,8 @@ fn append_text_blocks(
) -> Result<(), OpenAiError> {
match value {
None | Some(Value::Null) => Ok(()),
- Some(Value::String(text)) if !text.is_empty() => {
+ Some(Value::String(text)) if text.is_empty() => Ok(()),
+ Some(Value::String(text)) => {
out.push(json!({"type":"text", "text":text}));
Ok(())
}
@@ -892,6 +911,7 @@ fn image_url_block(url: &str, param: &str) -> Result<Value, OpenAiError> {
if encoded.len() > MAX_IMAGE_DATA_BYTES * 4 / 3 + 4 {
return Err(OpenAiError::invalid("Image data is too large", Some(param)));
}
+ let media_type = media_type.split(';').next().unwrap_or(media_type);
if !matches!(
media_type,
"image/png" | "image/jpeg" | "image/gif" | "image/webp"
@@ -918,6 +938,7 @@ fn image_url_block(url: &str, param: &str) -> Result<Value, OpenAiError> {
fn validate_cursor(
provider: &str,
session_id: Option<&str>,
+ stream: bool,
messages: &[Message],
tools: &[Value],
) -> Result<(), OpenAiError> {
@@ -935,6 +956,12 @@ fn validate_cursor(
if tools.is_empty() {
return Ok(());
}
+ if !stream {
+ return Err(OpenAiError::invalid(
+ "Cursor tools require 'stream' to be true",
+ Some("stream"),
+ ));
+ }
if session_id.is_none_or(str::is_empty) {
return Err(OpenAiError::invalid(
"Cursor tools require a stable session header",
@@ -1129,6 +1156,7 @@ mod tests {
OpenAiSurface::ChatCompletions,
json!({
"model":"cursor:gpt-5.5",
+ "stream":true,
"messages":[{"role":"user","content":"x"}],
"tools":[{"type":"function","function":{"name":"Read","parameters":{"type":"object"}}}]
}),
@@ -1139,6 +1167,47 @@ mod tests {
assert_eq!(error.param.as_deref(), Some("tools"));
}
+ #[test]
+ fn rejects_duplicate_results_store_and_buffered_cursor_tools() {
+ let duplicate = parse_request(
+ OpenAiSurface::ChatCompletions,
+ json!({
+ "model":"kimi-k2.6",
+ "messages":[
+ {"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"lookup","arguments":"{}"}}]},
+ {"role":"tool","tool_call_id":"call_1","content":"first"},
+ {"role":"tool","tool_call_id":"call_1","content":"second"}
+ ]
+ }),
+ "kimi",
+ Some("session"),
+ )
+ .unwrap_err();
+ assert!(duplicate.message.contains("unknown call"));
+
+ let store = parse_request(
+ OpenAiSurface::Responses,
+ json!({"model":"grok-4.5","input":"hello","store":true}),
+ "grok",
+ None,
+ )
+ .unwrap_err();
+ assert_eq!(store.param.as_deref(), Some("store"));
+
+ let cursor = parse_request(
+ OpenAiSurface::ChatCompletions,
+ json!({
+ "model":"cursor:gpt-5.5",
+ "messages":[{"role":"user","content":"x"}],
+ "tools":[{"type":"function","function":{"name":"Read","parameters":{"type":"object"}}}]
+ }),
+ "cursor",
+ Some("session"),
+ )
+ .unwrap_err();
+ assert_eq!(cursor.param.as_deref(), Some("stream"));
+ }
+
#[test]
fn normalizes_model_and_preserves_requested_value() {
let parsed = parse_request(
diff --git a/src/openai_compat/response.rs b/src/openai_compat/response.rs
index 141313d..1bee815 100644
--- a/src/openai_compat/response.rs
+++ b/src/openai_compat/response.rs
@@ -2,6 +2,10 @@ use serde_json::{Value, json};
use super::{OpenAiError, OpenAiSurface};
+fn upstream_invalid(message: impl Into<String>, _param: Option<impl Into<String>>) -> OpenAiError {
+ OpenAiError::upstream_protocol(message)
+}
+
#[derive(Debug, Clone)]
pub struct SseEvent {
pub event: Option<String>,
@@ -86,7 +90,7 @@ impl AnthropicAccumulator {
match kind {
"message_start" => {
let message = event.data.get("message").ok_or_else(|| {
- OpenAiError::invalid(
+ upstream_invalid(
"Provider message_start is missing 'message'",
None::<String>,
)
@@ -104,7 +108,7 @@ impl AnthropicAccumulator {
"content_block_start" => {
let index = required_index(&event.data)?;
let block = event.data.get("content_block").ok_or_else(|| {
- OpenAiError::invalid("Provider content block is missing", None::<String>)
+ upstream_invalid("Provider content block is missing", None::<String>)
})?;
let kind = match block.get("type").and_then(Value::as_str) {
Some("text") => BlockKind::Text {
@@ -138,13 +142,13 @@ impl AnthropicAccumulator {
},
Some(kind) if kind.ends_with("_tool_result") => BlockKind::HostedResult,
Some(other) => {
- return Err(OpenAiError::invalid(
+ return Err(upstream_invalid(
format!("Unsupported provider output block '{other}'"),
None::<String>,
));
}
None => {
- return Err(OpenAiError::invalid(
+ return Err(upstream_invalid(
"Provider output block has no type",
None::<String>,
));
@@ -155,7 +159,7 @@ impl AnthropicAccumulator {
"content_block_delta" => {
let index = required_index(&event.data)?;
let delta = event.data.get("delta").ok_or_else(|| {
- OpenAiError::invalid("Provider content delta is missing", None::<String>)
+ upstream_invalid("Provider content delta is missing", None::<String>)
})?;
let block = self
.blocks
@@ -163,7 +167,7 @@ impl AnthropicAccumulator {
.rev()
.find(|block| block.index == index)
.ok_or_else(|| {
- OpenAiError::invalid(
+ upstream_invalid(
"Provider delta references an unknown block",
None::<String>,
)
@@ -200,18 +204,36 @@ impl AnthropicAccumulator {
}
(Some("citations_delta"), BlockKind::Text { .. }) => {
if let Some(citation) = delta.get("citation") {
- self.citations.push(openai_citation(citation));
+ self.citations.push(citation.clone());
}
}
_ => {
- return Err(OpenAiError::invalid(
+ return Err(upstream_invalid(
"Provider delta does not match its content block",
None::<String>,
));
}
}
}
- "content_block_stop" => {}
+ "content_block_stop" => {
+ let index = required_index(&event.data)?;
+ if let Some(arguments) = self.blocks.iter().find_map(|block| {
+ if block.index != index {
+ return None;
+ }
+ match &block.kind {
+ BlockKind::Tool { arguments, .. }
+ | BlockKind::HostedSearch { arguments, .. } => Some(arguments),
+ _ => None,
+ }
+ }) && !arguments.is_empty()
+ && serde_json::from_str::<Value>(arguments).is_err()
+ {
+ return Err(OpenAiError::upstream_protocol(
+ "Provider emitted malformed tool arguments",
+ ));
+ }
+ }
"message_delta" => {
self.stop_reason = event
.data
@@ -243,7 +265,7 @@ impl AnthropicAccumulator {
});
}
other if !other.is_empty() => {
- return Err(OpenAiError::invalid(
+ return Err(upstream_invalid(
format!("Unsupported provider stream event '{other}'"),
None::<String>,
));
@@ -285,7 +307,7 @@ pub fn buffered_response(
state.apply(event)?;
}
if !state.stopped {
- return Err(OpenAiError::invalid(
+ return Err(upstream_invalid(
"Provider stream ended before message_stop",
None::<String>,
));
@@ -355,7 +377,7 @@ pub fn chat_response(
if !state.citations.is_empty() {
message.insert(
"annotations".to_string(),
- Value::Array(state.citations.clone()),
+ Value::Array(state.citations.iter().map(chat_citation).collect()),
);
}
json!({
@@ -380,47 +402,31 @@ pub fn responses_response(
created: u64,
) -> Value {
let mut output = Vec::new();
- let reasoning = state
- .blocks
- .iter()
- .filter_map(|block| match &block.kind {
- BlockKind::Thinking { text } => Some(text.as_str()),
- _ => None,
- })
- .collect::<String>();
- if !reasoning.is_empty() {
- output.push(json!({
- "id":format!("rs_{}", stable_suffix(response_id, 0)),
- "type":"reasoning",
- "summary":[{"type":"summary_text", "text":reasoning}],
- "status":"completed",
- }));
- }
- let text = state
+ for block in state
.blocks
.iter()
- .filter_map(|block| match &block.kind {
- BlockKind::Text { text } => Some(text.as_str()),
- _ => None,
- })
- .collect::<String>();
- if !text.is_empty() {
- output.push(json!({
- "id":format!("msg_{}", stable_suffix(response_id, 1)),
- "type":"message",
- "role":"assistant",
- "status":"completed",
- "content":[{"type":"output_text", "text":text, "annotations":state.citations}],
- }));
- }
- for (index, block) in state.blocks.iter().enumerate() {
+ .filter(|block| !matches!(block.kind, BlockKind::HostedResult))
+ {
match &block.kind {
+ BlockKind::Text { text } => output.push(json!({
+ "id":format!("msg_{}", stable_suffix(response_id, block.index)),
+ "type":"message",
+ "role":"assistant",
+ "status":"completed",
+ "content":[{"type":"output_text", "text":text, "annotations":state.citations.iter().map(responses_citation).collect::<Vec<_>>() }],
+ })),
+ BlockKind::Thinking { text } => output.push(json!({
+ "id":format!("rs_{}", stable_suffix(response_id, block.index)),
+ "type":"reasoning",
+ "summary":[{"type":"summary_text", "text":text}],
+ "status":"completed",
+ })),
BlockKind::Tool {
id,
name,
arguments,
} => output.push(json!({
- "id":format!("fc_{}", stable_suffix(response_id, index + 2)),
+ "id":format!("fc_{}", stable_suffix(response_id, block.index)),
"type":"function_call",
"call_id":id,
"name":name,
@@ -437,19 +443,20 @@ pub fn responses_response(
"status":"completed",
"action":hosted_search_action(name, arguments),
})),
- _ => {}
+ BlockKind::HostedResult => unreachable!(),
}
}
+ let incomplete = state.stop_reason.as_deref() == Some("max_tokens");
json!({
"id":response_id,
"object":"response",
"created_at":created,
- "status":"completed",
+ "status":if incomplete { "incomplete" } else { "completed" },
"model":model,
"output":output,
"parallel_tool_calls":false,
"error":null,
- "incomplete_details":null,
+ "incomplete_details":if incomplete { json!({"reason":"max_output_tokens"}) } else { Value::Null },
"usage":state.usage.responses_value(),
})
}
@@ -475,25 +482,33 @@ pub fn hosted_search_action(_name: &str, arguments: &str) -> Value {
json!({"type":"search", "query":query})
}
-fn openai_citation(citation: &Value) -> Value {
+pub fn chat_citation(citation: &Value) -> Value {
json!({
"type":"url_citation",
"url_citation":{
"url":citation.get("url").and_then(Value::as_str).unwrap_or_default(),
"title":citation.get("title").and_then(Value::as_str).unwrap_or_default(),
- "start_index":0,
- "end_index":0,
+ "start_index":citation.get("start_index").and_then(Value::as_u64).unwrap_or_default(),
+ "end_index":citation.get("end_index").and_then(Value::as_u64).unwrap_or_default(),
}
})
}
+pub fn responses_citation(citation: &Value) -> Value {
+ json!({
+ "type":"url_citation",
+ "url":citation.get("url").and_then(Value::as_str).unwrap_or_default(),
+ "title":citation.get("title").and_then(Value::as_str).unwrap_or_default(),
+ "start_index":citation.get("start_index").and_then(Value::as_u64).unwrap_or_default(),
+ "end_index":citation.get("end_index").and_then(Value::as_u64).unwrap_or_default(),
+ })
+}
+
pub fn normalized_arguments(arguments: &str) -> String {
if arguments.is_empty() {
"{}".to_string()
- } else if serde_json::from_str::<Value>(arguments).is_ok() {
- arguments.to_string()
} else {
- json!({"value":arguments}).to_string()
+ arguments.to_string()
}
}
@@ -506,7 +521,7 @@ fn required_index(value: &Value) -> Result<usize, OpenAiError> {
.get("index")
.and_then(Value::as_u64)
.and_then(|value| usize::try_from(value).ok())
- .ok_or_else(|| OpenAiError::invalid("Provider event has an invalid index", None::<String>))
+ .ok_or_else(|| upstream_invalid("Provider event has an invalid index", None::<String>))
}
fn required_block_string(block: &Value, key: &str) -> Result<String, OpenAiError> {
@@ -516,7 +531,7 @@ fn required_block_string(block: &Value, key: &str) -> Result<String, OpenAiError
.filter(|value| !value.is_empty())
.map(str::to_string)
.ok_or_else(|| {
- OpenAiError::invalid(
+ upstream_invalid(
format!("Provider tool block has an invalid '{key}'"),
None::<String>,
)
diff --git a/src/openai_compat/stream.rs b/src/openai_compat/stream.rs
index 4ead47a..6cceb6f 100644
--- a/src/openai_compat/stream.rs
+++ b/src/openai_compat/stream.rs
@@ -19,11 +19,16 @@ use crate::{
use super::{
MAX_PROVIDER_STREAM_BYTES, MAX_SSE_EVENT_BYTES, OpenAiError, OpenAiSurface,
response::{
- AnthropicAccumulator, BlockKind, SseEvent, buffered_response, chat_finish_reason,
- hosted_search_action, normalized_arguments, responses_response,
+ AnthropicAccumulator, BlockKind, SseEvent, buffered_response, chat_citation,
+ chat_finish_reason, hosted_search_action, normalized_arguments, responses_citation,
+ responses_response,
},
};
+fn upstream_invalid(message: impl Into<String>, _param: Option<impl Into<String>>) -> OpenAiError {
+ OpenAiError::upstream_protocol(message)
+}
+
#[derive(Default)]
pub struct SseDecoder {
pending: BytesMut,
@@ -32,20 +37,24 @@ pub struct SseDecoder {
impl SseDecoder {
pub fn push(&mut self, bytes: &[u8]) -> Result<Vec<SseEvent>, OpenAiError> {
self.pending.extend_from_slice(bytes);
- if self.pending.len() > MAX_SSE_EVENT_BYTES {
- return Err(OpenAiError::invalid(
- "Provider SSE event exceeded the size limit",
- None::<String>,
- ));
- }
let mut events = Vec::new();
while let Some((position, delimiter_len)) = find_event_delimiter(&self.pending) {
+ if position > MAX_SSE_EVENT_BYTES {
+ return Err(OpenAiError::upstream_protocol(
+ "Provider SSE event exceeded the size limit",
+ ));
+ }
let frame = self.pending.split_to(position + delimiter_len);
let payload = &frame[..position];
if let Some(event) = parse_frame(payload)? {
events.push(event);
}
}
+ if self.pending.len() > MAX_SSE_EVENT_BYTES {
+ return Err(OpenAiError::upstream_protocol(
+ "Provider SSE event exceeded the size limit",
+ ));
+ }
Ok(events)
}
@@ -53,9 +62,8 @@ impl SseDecoder {
if self.pending.iter().all(u8::is_ascii_whitespace) {
Ok(())
} else {
- Err(OpenAiError::invalid(
+ Err(OpenAiError::upstream_protocol(
"Provider SSE stream ended with an incomplete event",
- None::<String>,
))
}
}
@@ -66,9 +74,9 @@ pub async fn openai_response(
generation: Generation,
stream: bool,
include_usage: bool,
- model: String,
traffic: Option<Arc<TrafficCapture>>,
) -> Result<Response, OpenAiError> {
+ let model = generation.resolved_model.clone();
let response_id = match surface {
OpenAiSurface::ChatCompletions => format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()),
OpenAiSurface::Responses => format!("resp_{}", uuid::Uuid::new_v4().simple()),
@@ -98,9 +106,8 @@ async fn collect_generation(body: GenerationBody) -> Result<Bytes, OpenAiError>
match body {
GenerationBody::BufferedSse(bytes) => {
if bytes.len() > MAX_PROVIDER_STREAM_BYTES {
- Err(OpenAiError::invalid(
+ Err(OpenAiError::upstream_protocol(
"Provider response exceeded the size limit",
- None::<String>,
))
} else {
Ok(bytes)
@@ -119,9 +126,8 @@ async fn collect_generation(body: GenerationBody) -> Result<Bytes, OpenAiError>
})?;
if let Ok(data) = frame.into_data() {
if output.len().saturating_add(data.len()) > MAX_PROVIDER_STREAM_BYTES {
- return Err(OpenAiError::invalid(
+ return Err(OpenAiError::upstream_protocol(
"Provider response exceeded the size limit",
- None::<String>,
));
}
output.extend_from_slice(&data);
@@ -221,9 +227,8 @@ impl StreamState {
};
self.bytes = self.bytes.saturating_add(data.len());
if self.bytes > MAX_PROVIDER_STREAM_BYTES {
- self.fail(OpenAiError::invalid(
+ self.fail(OpenAiError::upstream_protocol(
"Provider response exceeded the size limit",
- None::<String>,
));
continue;
}
@@ -254,7 +259,7 @@ impl StreamState {
if let Err(error) = self.decoder.finish() {
self.fail(error);
} else if !self.renderer.state.stopped {
- self.fail(OpenAiError::invalid(
+ self.fail(upstream_invalid(
"Provider stream ended before message_stop",
None::<String>,
));
@@ -267,9 +272,10 @@ impl StreamState {
}
fn capture_bytes(&mut self, bytes: &[u8]) {
- if self.downstream.len().saturating_add(bytes.len()) <= MAX_SSE_CAPTURE_BYTES {
- self.downstream.extend_from_slice(bytes);
- } else {
+ let remaining = MAX_SSE_CAPTURE_BYTES.saturating_sub(self.downstream.len());
+ self.downstream
+ .extend_from_slice(&bytes[..bytes.len().min(remaining)]);
+ if bytes.len() > remaining {
self.downstream_truncated = true;
}
}
@@ -315,7 +321,6 @@ struct Renderer {
created: u64,
sequence: u64,
state: AnthropicAccumulator,
- responses_message_started: bool,
}
impl Renderer {
@@ -334,7 +339,6 @@ impl Renderer {
created,
sequence: 0,
state: AnthropicAccumulator::default(),
- responses_message_started: false,
}
}
@@ -366,12 +370,13 @@ impl Renderer {
if let Some(block) = self.state.blocks.iter().rev().find(|block| block.index == index as usize)
&& let BlockKind::Tool { id, name, .. } = &block.kind
{
+ let tool_index = self.tool_index(index as usize);
out.push(chat_data(json!({
"id":self.response_id,
"object":"chat.completion.chunk",
"created":self.created,
"model":self.model,
- "choices":[{"index":0,"delta":{"tool_calls":[{"index":index,"id":id,"type":"function","function":{"name":name,"arguments":""}}]},"finish_reason":null,"logprobs":null}],
+ "choices":[{"index":0,"delta":{"tool_calls":[{"index":tool_index,"id":id,"type":"function","function":{"name":name,"arguments":""}}]},"finish_reason":null,"logprobs":null}],
})));
}
}
@@ -386,14 +391,15 @@ impl Renderer {
block.index == index as usize
&& matches!(block.kind, BlockKind::Tool { .. })
}) {
- json!({"tool_calls":[{"index":index,"function":{"arguments":delta.get("partial_json").and_then(Value::as_str).unwrap_or_default()}}]})
+ let tool_index = self.tool_index(index as usize);
+ json!({"tool_calls":[{"index":tool_index,"function":{"arguments":delta.get("partial_json").and_then(Value::as_str).unwrap_or_default()}}]})
} else {
return Ok(out);
}
}
- Some("citations_delta") => json!({"annotations":self.state.citations.last().cloned().into_iter().collect::<Vec<_>>()}),
+ Some("citations_delta") => json!({"annotations":self.state.citations.iter().map(responses_citation).collect::<Vec<_>>().last().map(chat_citation).into_iter().collect::<Vec<_>>()}),
Some("signature_delta") => return Ok(out),
- _ => return Err(OpenAiError::invalid("Unsupported provider content delta", None::<String>)),
+ _ => return Err(upstream_invalid("Unsupported provider content delta", None::<String>)),
};
out.push(chat_data(json!({
"id":self.response_id,
@@ -444,6 +450,7 @@ impl Renderer {
.get("index")
.and_then(Value::as_u64)
.unwrap_or_default() as usize;
+ let output_index = self.output_index(index);
let block = self
.state
.blocks
@@ -452,33 +459,36 @@ impl Renderer {
.find(|block| block.index == index)
.cloned()
.ok_or_else(|| {
- OpenAiError::invalid("Provider content block is missing", None::<String>)
+ upstream_invalid("Provider content block is missing", None::<String>)
})?;
match &block.kind {
BlockKind::Text { .. } => {
- let item_id = self.message_item_id();
- if !self.responses_message_started {
- out.push(self.responses_event("response.output_item.added", json!({
- "output_index":index,
- "item":{"id":item_id,"type":"message","role":"assistant","status":"in_progress","content":[]},
- })));
- self.responses_message_started = true;
- }
+ let item_id = self.message_item_id(index);
+ out.push(self.responses_event("response.output_item.added", json!({
+ "output_index":output_index,
+ "item":{"id":item_id,"type":"message","role":"assistant","status":"in_progress","content":[]},
+ })));
out.push(self.responses_event("response.content_part.added", json!({
- "item_id":item_id,"output_index":index,"content_index":0,
+ "item_id":item_id,"output_index":output_index,"content_index":0,
"part":{"type":"output_text","text":"","annotations":[]},
})));
}
- BlockKind::Thinking { .. } => out.push(self.responses_event("response.output_item.added", json!({
- "output_index":index,
- "item":{"id":self.block_item_id("rs", index),"type":"reasoning","status":"in_progress","summary":[]},
- }))),
+ BlockKind::Thinking { .. } => {
+ out.push(self.responses_event("response.output_item.added", json!({
+ "output_index":output_index,
+ "item":{"id":self.block_item_id("rs", index),"type":"reasoning","status":"in_progress","summary":[]},
+ })));
+ out.push(self.responses_event("response.reasoning_summary_part.added", json!({
+ "item_id":self.block_item_id("rs", index),"output_index":output_index,"summary_index":0,
+ "part":{"type":"summary_text","text":""},
+ })));
+ }
BlockKind::Tool { id, name, .. } => out.push(self.responses_event("response.output_item.added", json!({
- "output_index":index,
+ "output_index":output_index,
"item":{"id":self.block_item_id("fc", index),"type":"function_call","call_id":id,"name":name,"arguments":"","status":"in_progress"},
}))),
BlockKind::HostedSearch { id, .. } => out.push(self.responses_event("response.output_item.added", json!({
- "output_index":index,
+ "output_index":output_index,
"item":{"id":id,"type":"web_search_call","status":"in_progress"},
}))),
BlockKind::HostedResult => {}
@@ -490,14 +500,15 @@ impl Renderer {
.get("index")
.and_then(Value::as_u64)
.unwrap_or_default() as usize;
+ let output_index = self.output_index(index);
let delta = event.data.get("delta").unwrap_or(&Value::Null);
match delta.get("type").and_then(Value::as_str) {
Some("text_delta") => out.push(self.responses_event("response.output_text.delta", json!({
- "item_id":self.message_item_id(),"output_index":index,"content_index":0,
+ "item_id":self.message_item_id(index),"output_index":output_index,"content_index":0,
"delta":delta.get("text").and_then(Value::as_str).unwrap_or_default(),"logprobs":[],
}))),
Some("thinking_delta") => out.push(self.responses_event("response.reasoning_summary_text.delta", json!({
- "item_id":self.block_item_id("rs", index),"output_index":index,"summary_index":0,
+ "item_id":self.block_item_id("rs", index),"output_index":output_index,"summary_index":0,
"delta":delta.get("thinking").and_then(Value::as_str).unwrap_or_default(),
}))),
Some("input_json_delta") => {
@@ -505,18 +516,18 @@ impl Renderer {
block.index == index && matches!(block.kind, BlockKind::Tool { .. })
}) {
out.push(self.responses_event("response.function_call_arguments.delta", json!({
- "item_id":self.block_item_id("fc", index),"output_index":index,
+ "item_id":self.block_item_id("fc", index),"output_index":output_index,
"delta":delta.get("partial_json").and_then(Value::as_str).unwrap_or_default(),
})));
}
}
Some("citations_delta") => out.push(self.responses_event("response.output_text.annotation.added", json!({
- "item_id":self.message_item_id(),"output_index":index,"content_index":0,
+ "item_id":self.message_item_id(index),"output_index":output_index,"content_index":0,
"annotation_index":self.state.citations.len().saturating_sub(1),
- "annotation":self.state.citations.last().cloned().unwrap_or(Value::Null),
+ "annotation":self.state.citations.last().map(responses_citation).unwrap_or(Value::Null),
}))),
Some("signature_delta") => {}
- _ => return Err(OpenAiError::invalid("Unsupported provider content delta", None::<String>)),
+ _ => return Err(upstream_invalid("Unsupported provider content delta", None::<String>)),
}
}
"content_block_stop" => {
@@ -525,6 +536,7 @@ impl Renderer {
.get("index")
.and_then(Value::as_u64)
.unwrap_or_default() as usize;
+ let output_index = self.output_index(index);
let block = self
.state
.blocks
@@ -533,28 +545,32 @@ impl Renderer {
.find(|block| block.index == index)
.cloned()
.ok_or_else(|| {
- OpenAiError::invalid("Provider content block is missing", None::<String>)
+ upstream_invalid("Provider content block is missing", None::<String>)
})?;
match block.kind {
BlockKind::Text { text } => {
out.push(self.responses_event("response.output_text.done", json!({
- "item_id":self.message_item_id(),"output_index":index,"content_index":0,"text":text,"logprobs":[],
+ "item_id":self.message_item_id(index),"output_index":output_index,"content_index":0,"text":text,"logprobs":[],
})));
out.push(self.responses_event("response.content_part.done", json!({
- "item_id":self.message_item_id(),"output_index":index,"content_index":0,
- "part":{"type":"output_text","text":text,"annotations":self.state.citations},
+ "item_id":self.message_item_id(index),"output_index":output_index,"content_index":0,
+ "part":{"type":"output_text","text":text,"annotations":self.state.citations.iter().map(responses_citation).collect::<Vec<_>>()},
})));
out.push(self.responses_event("response.output_item.done", json!({
- "output_index":index,
- "item":{"id":self.message_item_id(),"type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":text,"annotations":self.state.citations}]},
+ "output_index":output_index,
+ "item":{"id":self.message_item_id(index),"type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":text,"annotations":self.state.citations.iter().map(responses_citation).collect::<Vec<_>>()}]},
})));
}
BlockKind::Thinking { text } => {
out.push(self.responses_event("response.reasoning_summary_text.done", json!({
- "item_id":self.block_item_id("rs", index),"output_index":index,"summary_index":0,"text":text,
+ "item_id":self.block_item_id("rs", index),"output_index":output_index,"summary_index":0,"text":text,
+ })));
+ out.push(self.responses_event("response.reasoning_summary_part.done", json!({
+ "item_id":self.block_item_id("rs", index),"output_index":output_index,"summary_index":0,
+ "part":{"type":"summary_text","text":text},
})));
out.push(self.responses_event("response.output_item.done", json!({
- "output_index":index,
+ "output_index":output_index,
"item":{"id":self.block_item_id("rs", index),"type":"reasoning","status":"completed","summary":[{"type":"summary_text","text":text}]},
})));
}
@@ -565,10 +581,10 @@ impl Renderer {
} => {
let arguments = normalized_arguments(&arguments);
out.push(self.responses_event("response.function_call_arguments.done", json!({
- "item_id":self.block_item_id("fc", index),"output_index":index,"arguments":arguments,
+ "item_id":self.block_item_id("fc", index),"output_index":output_index,"arguments":arguments,
})));
out.push(self.responses_event("response.output_item.done", json!({
- "output_index":index,
+ "output_index":output_index,
"item":{"id":self.block_item_id("fc", index),"type":"function_call","call_id":id,"name":name,"arguments":arguments,"status":"completed"},
})));
}
@@ -577,7 +593,7 @@ impl Renderer {
name,
arguments,
} => out.push(self.responses_event("response.output_item.done", json!({
- "output_index":index,
+ "output_index":output_index,
"item":{"id":id,"type":"web_search_call","status":"completed","action":hosted_search_action(&name, &arguments)},
}))),
BlockKind::HostedResult => {}
@@ -586,7 +602,12 @@ impl Renderer {
"message_stop" => {
let response =
responses_response(&self.state, &self.response_id, &self.model, self.created);
- out.push(self.responses_event("response.completed", json!({"response":response})));
+ let kind = if self.state.stop_reason.as_deref() == Some("max_tokens") {
+ "response.incomplete"
+ } else {
+ "response.completed"
+ };
+ out.push(self.responses_event(kind, json!({"response":response})));
}
_ => {}
}
@@ -595,9 +616,12 @@ impl Renderer {
fn failure(&mut self, error: &OpenAiError) -> Vec<Bytes> {
match self.surface {
- OpenAiSurface::ChatCompletions => vec![chat_data(json!({
- "error":{"message":error.message,"type":error.kind,"param":error.param,"code":error.code}
- }))],
+ OpenAiSurface::ChatCompletions => vec![
+ chat_data(json!({
+ "error":{"message":error.message,"type":error.kind,"param":error.param,"code":error.code}
+ })),
+ Bytes::from_static(b"data: [DONE]\n\n"),
+ ],
OpenAiSurface::Responses => {
let mut response =
response_shell(&self.response_id, &self.model, self.created, "failed");
@@ -616,8 +640,28 @@ impl Renderer {
named_sse(kind, Value::Object(value))
}
- fn message_item_id(&self) -> String {
- format!("msg_{}", self.response_id.trim_start_matches("resp_"))
+ fn message_item_id(&self, block_index: usize) -> String {
+ self.block_item_id("msg", block_index)
+ }
+
+ fn output_index(&self, block_index: usize) -> usize {
+ self.state
+ .blocks
+ .iter()
+ .filter(|block| {
+ block.index < block_index && !matches!(block.kind, BlockKind::HostedResult)
+ })
+ .count()
+ }
+
+ fn tool_index(&self, block_index: usize) -> usize {
+ self.state
+ .blocks
+ .iter()
+ .filter(|block| {
+ block.index < block_index && matches!(block.kind, BlockKind::Tool { .. })
+ })
+ .count()
}
fn block_item_id(&self, prefix: &str, index: usize) -> String {
@@ -653,7 +697,7 @@ fn response_shell(id: &str, model: &str, created: u64, status: &str) -> Value {
fn parse_frame(frame: &[u8]) -> Result<Option<SseEvent>, OpenAiError> {
let text = std::str::from_utf8(frame)
- .map_err(|_| OpenAiError::invalid("Provider SSE event is not UTF-8", None::<String>))?;
+ .map_err(|_| OpenAiError::upstream_protocol("Provider SSE event is not UTF-8"))?;
let mut event = None;
let mut data = Vec::new();
for line in text.lines() {
@@ -675,10 +719,7 @@ fn parse_frame(frame: &[u8]) -> Result<Option<SseEvent>, OpenAiError> {
return Ok(None);
}
let data = serde_json::from_str(&data).map_err(|error| {
- OpenAiError::invalid(
- format!("Provider SSE event contains invalid JSON: {error}"),
- None::<String>,
- )
+ OpenAiError::upstream_protocol(format!("Provider SSE event contains invalid JSON: {error}"))
})?;
Ok(Some(SseEvent { event, data }))
}
@@ -726,6 +767,16 @@ mod tests {
decoder.finish().unwrap();
}
+ #[test]
+ fn decoder_accepts_large_batches_of_small_events() {
+ let event = "event: ping\ndata: {\"type\":\"ping\"}\n\n";
+ let count = MAX_SSE_EVENT_BYTES / event.len() + 1;
+ let mut decoder = SseDecoder::default();
+ let events = decoder.push(event.repeat(count).as_bytes()).unwrap();
+ assert_eq!(events.len(), count);
+ decoder.finish().unwrap();
+ }
+
#[tokio::test]
async fn chat_stream_emits_tool_deltas_usage_and_done() {
let input = concat!(
diff --git a/src/server.rs b/src/server.rs
index 7210b48..a4364dd 100644
--- a/src/server.rs
+++ b/src/server.rs
@@ -615,13 +615,13 @@ async fn handler_responses(State(state): State<Arc<AppState>>, req: Request<Body
let request_guard = RequestMonitorGuard::new(state.monitor.clone(), req_id.clone());
let body_bytes = match axum::body::to_bytes(req.into_body(), MAX_OPENAI_REQUEST_BYTES).await {
Ok(bytes) => bytes,
- Err(error) => {
+ Err(_) => {
let response = openai_error(
- StatusCode::BAD_REQUEST,
+ StatusCode::PAYLOAD_TOO_LARGE,
"invalid_request_error",
- format!("Invalid JSON: {error}"),
+ "Request body exceeded the size limit".to_string(),
None,
- Some("invalid_json"),
+ Some("request_too_large"),
);
log_native_request_completed(
&log,
@@ -778,7 +778,6 @@ async fn handler_responses(State(state): State<Arc<AppState>>, req: Request<Body
generation,
parsed.stream,
parsed.include_usage,
- normalized_model.clone(),
traffic.clone(),
)
.await
@@ -847,13 +846,13 @@ async fn handler_chat_completions(
let request_guard = RequestMonitorGuard::new(state.monitor.clone(), req_id.clone());
let body_bytes = match axum::body::to_bytes(req.into_body(), MAX_OPENAI_REQUEST_BYTES).await {
Ok(bytes) => bytes,
- Err(error) => {
+ Err(_) => {
let response = openai_error(
- StatusCode::BAD_REQUEST,
+ StatusCode::PAYLOAD_TOO_LARGE,
"invalid_request_error",
- format!("Invalid JSON: {error}"),
+ "Request body exceeded the size limit".to_string(),
None,
- Some("invalid_json"),
+ Some("request_too_large"),
);
log_native_request_completed(
&log,
@@ -1030,7 +1029,6 @@ async fn handler_chat_completions(
generation,
parsed.stream,
parsed.include_usage,
- normalized_model.clone(),
traffic.clone(),
)
.await