Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): relay OpenAI Chat Completions with usage telemetry (#18204)

Hassieb Pakzad62ac0f03ad76
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.toml
index 3a8edbc883..8d16d2ff39 100644
--- a/ai-gateway/Cargo.toml
+++ b/ai-gateway/Cargo.toml
@@ -21,7 +21,7 @@ reqwest = { version = "0.13.5", default-features = false, features = ["rustls",
reqwest-middleware = "0.5"
reqwest-tracing = { version = "0.7", default-features = false }
serde = { version = "1", features = ["derive"] }
-serde_json = "1"
+serde_json = { version = "1", features = ["raw_value"] }
sha2 = "0.11"
tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }
tower-http = { version = "0.7", features = ["trace"] }
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index ac2b37ed81..cb7f04150b 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -4,11 +4,12 @@ A standalone Rust gateway for native provider APIs under two namespaces:
| Namespace | Endpoints | Generation ingested |
| --- | --- | --- |
-| `/openai/v1` | `POST /responses`, `POST /responses/compact`, `GET /models` | Responses and compact |
+| `/openai/v1` | `POST /responses`, `POST /responses/compact`, `POST /chat/completions`, `GET /models` | Responses, compact and Chat Completions |
| `/anthropic/v1` | `POST /messages`, `POST /messages/count_tokens`, `GET /models` | Messages |
Web resolves the gateway key to a trusted provider connection for the namespace's API
-format; Rust relays native JSON or SSE without rewriting provider bytes. Models listings
+format; Rust relays native JSON or SSE without rewriting provider bytes, except that a
+streamed Chat Completions request is asked for its usage (see below). Models listings
are GET proxies of the provider catalog and token counting returns the native count;
neither is ingested as a generation. A bounded capture layer captures request/response
facts and logs capture completeness at debug level. Each finalized generation is
@@ -155,7 +156,8 @@ metadata; it is not forwarded to Web or the provider. The provider's own request
generation metadata, and its response header is still relayed unchanged.
When the request will produce a Langfuse generation (`/openai/v1/responses`,
-`/openai/v1/responses/compact` and `/anthropic/v1/messages` with inference telemetry
+`/openai/v1/responses/compact`, `/openai/v1/chat/completions` and
+`/anthropic/v1/messages` with inference telemetry
configured), the response also carries `langfuse-trace-id` and
`langfuse-observation-id`: the uploaded generation's trace ID, after `traceparent`
and agent turn grouping, and its OpenTelemetry span ID, which Langfuse stores as the
@@ -234,6 +236,11 @@ curl http://localhost:8080/openai/v1/responses/compact \
-H 'Content-Type: application/json' \
-d '{"model":"gpt-4.1","input":[]}'
+curl http://localhost:8080/openai/v1/chat/completions \
+ -H "Authorization: Bearer $LANGFUSE_GATEWAY_KEY" \
+ -H 'Content-Type: application/json' \
+ -d '{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"Say hello"}]}'
+
curl http://localhost:8080/openai/v1/models \
-H "Authorization: Bearer $LANGFUSE_GATEWAY_KEY"
```
@@ -249,8 +256,28 @@ client = OpenAI(
)
response = client.responses.create(model="gpt-4.1-mini", input="Say hello")
print(response.output_text)
+completion = client.chat.completions.create(
+ model="gpt-4.1-mini", messages=[{"role": "user", "content": "Say hello"}]
+)
+print(completion.choices[0].message.content)
```
+### Chat Completions streaming usage
+
+OpenAI reports usage for a streamed Chat Completions request only when the request
+sets `stream_options.include_usage`. When a request has `"stream":true` and does not
+set it to `true`, the gateway adds `"include_usage":true` to the upstream copy so the
+generation still gets token usage and cost; other `stream_options` fields and every
+other top-level member keep their original bytes and order. The recorded input and
+model parameters are the caller's own request. The response stream is relayed byte
+for byte, so a client that did not request usage still receives OpenAI's final usage
+chunk: an extra event with an empty `choices` array and a `usage` object, just before
+`data: [DONE]`. The other chunks of such a stream also carry `"usage":null`. Clients
+must tolerate a chunk without choices, as they already must when they request usage
+themselves. Requests that already ask for usage, content-encoded bodies, ambiguous
+bodies (duplicate `stream` or `stream_options` members) and non-streamed requests are
+forwarded unchanged.
+
### Anthropic Messages and Claude Code
The `/anthropic/v1` namespace serves the Anthropic Messages API. Claude Code treats the
@@ -389,8 +416,9 @@ attribution. Full mode includes the captured input/output; usage mode omits cont
Completion-start time is emitted only for upstream `text/event-stream` responses,
on the first nonempty text, reasoning, refusal, tool-input, audio, or partial-image content. JSON responses
and streams without a captured content delta have no completion-start time or TTFT.
-OpenAI Responses usage is projected into the receiver's strict native schema, keeping
-its nested detail counters. Anthropic usage is uploaded exactly as the provider reported
+OpenAI Responses and Chat Completions usage are projected into the receiver's strict
+native schemas (`input_tokens`/`output_tokens` or `prompt_tokens`/`completion_tokens`,
+plus `total_tokens`), keeping their nested detail counters. Anthropic usage is uploaded exactly as the provider reported
it; ingestion decides what to price. Usage is not duplicated in metadata. Missing usage
is not reported as zero.
@@ -402,7 +430,7 @@ each field describes:
| `organization.id`, `project.id` | Resolved Langfuse organization and ingestion project |
| `api_key.id`, `connection.id` | Authenticated gateway key and selected Langfuse connection |
| `ingestion.mode` | Effective capture mode |
-| `request.api_format` | Native API contract: `openai.responses` or `anthropic.messages` |
+| `request.api_format` | Native API contract: `openai.responses`, `openai.chat-completions` or `anthropic.messages` |
| `request.metadata`, `request.prompt_cache_key`, `request.safety_identifier`, `request.user` | Native caller-supplied fields, captured only in full mode |
| `response.id` | Native response object's ID |
| `response.status_code` | HTTP status returned to the caller |
@@ -599,7 +627,8 @@ stored under `langfuse.gateway.request.*` only in full mode.
In both modes, `usage_details` preserves the provider's entire usage object,
including nested and unknown fields. The gateway does not rename counters,
subtract cached tokens, or synthesize totals. Missing usage stays null.
-`api_format` identifies the payload format (`openai.responses` or `anthropic.messages`).
+`api_format` identifies the payload format (`openai.responses`,
+`openai.chat-completions` or `anthropic.messages`); it is also the generation's name.
For Anthropic Messages, `message_start` supplies the response ID, actual model and the
usage baseline; `message_delta` supplies `stop_reason` and cumulative usage counters that
@@ -628,6 +657,28 @@ stopped blocks are recorded; an unfinished block, tool input that is not valid J
unknown delta type or a block past the budget is left out and makes the output
partial. Usage mode records neither input nor output.
+For OpenAI Chat Completions, scalar model parameters (`temperature`, `top_p`, `n`,
+`max_tokens`, `max_completion_tokens`, `presence_penalty`, `frequency_penalty`, `seed`,
+`logprobs`, `top_logprobs`, `service_tier`, `parallel_tool_calls`, `reasoning_effort`,
+`verbosity`, `stream`, `store`, `prompt_cache_retention`) are recorded in both modes.
+In full mode `stop`, `response_format`, `tool_choice`, `function_call`, `stream_options`,
+`logit_bias`, `modalities`, `audio`, `prediction` and `web_search_options` also become
+model parameters, `metadata`, `prompt_cache_key`, `safety_identifier` and `user` become
+`langfuse.gateway.request.*`, and `messages`, `tools`, legacy `functions` and unknown
+fields remain the input. The output is the native `{"choices":[...]}` list: JSON
+responses contribute their choices at EOF; streams rebuild each choice from its
+`delta` chunks. Text fragments (`content`, `refusal`, tool and function `arguments`,
+audio data) are appended, `tool_calls` are merged by their `index`, `role`, `id` and
+`type` are kept once, and each choice records its `finish_reason`. Each chunk supplies
+the response ID, actual model and service tier; the usage chunk supplies usage.
+`data: [DONE]` is the terminal event, and every choice needs a `finish_reason` for
+full-mode output to be complete. A choice whose fragments exceed the retained-output
+budget is left out and makes the output partial. `completion_start_ms` is set by the
+first delta with non-empty text, refusal, tool or function arguments, or audio; role
+headers and tool names do not count. A `finish_reason` of `length` or
+`content_filter` sets a `WARNING` level, and an `error` object in the body or a chunk
+marks the generation failed, with its `code` and `message` retained only in full mode.
+
For OpenAI Responses SSE, only `response.output_item.done` adds output. Terminal Responses events
provide model, service tier, status and usage; their repeated output is not copied.
Deltas are never stitched or retained. A disconnect midway through an item loses
@@ -652,7 +703,8 @@ content does not make it false. Request inspection failure does not invalidate a
fully captured response, and downstream cancellation does not erase completed capture.
The implementation separates shared `ExecutionCapture` lifecycle/timing, the
-`OpenAiResponsesCapture` and `AnthropicMessagesCapture` adapters, the shared
+`OpenAiResponsesCapture`, `OpenAiChatCompletionsCapture` and `AnthropicMessagesCapture`
+adapters, the shared
`ResponseBody` content-type sniffing and bounded `SseDecoder`. A private
`ProtocolCapture` enum dispatches to the adapter selected by API format. Finalization
hands an owned `InferenceFacts` record to telemetry for a safe debug summary and
@@ -746,7 +798,8 @@ async fn example(web_base_url: &str, service_key: &str, gateway_key: &str)
Each call sends `POST <base path>/api/internal/ai-gateway/v1/resolve` with
`Authorization: Bearer <gateway key>`, the Web v1 HMAC header, and exactly
`{"apiFormat":"<format>"}`, where the format is the one the public route serves
-(`openai.responses` for every `/openai/v1` route). The client neither receives nor
+(`openai.chat-completions` for `/openai/v1/chat/completions`, `openai.responses` for
+the other `/openai/v1` routes). The client neither receives nor
parses an inference request body. Reuse the resolver across requests: its connection
pool is shared, while credentials and execution contexts stay request-local.
@@ -766,7 +819,7 @@ invalid response, even if Web selected it.
| Provider | API format | Official origin | Credential |
| --- | --- | --- | --- |
-| `openai` | `openai.responses` | `https://api.openai.com/v1` | `{"type":"Bearer","token":…}` → `Authorization: Bearer` |
+| `openai` | `openai.responses`, `openai.chat-completions` | `https://api.openai.com/v1` | `{"type":"Bearer","token":…}` → `Authorization: Bearer` |
| `anthropic` | `anthropic.messages` | `https://api.anthropic.com/v1` | `{"type":"x-api-key","header":"x-api-key","value":…}` → `x-api-key` |
`ProviderConnection::credential()` exposes the secret in its header position
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rs
index 2f4c02f95a..dfedf15781 100644
--- a/ai-gateway/src/capture/mod.rs
+++ b/ai-gateway/src/capture/mod.rs
@@ -1,5 +1,6 @@
mod anthropic_messages;
mod facts;
+mod openai_chat_completions;
mod openai_responses;
mod response;
mod sse;
@@ -18,6 +19,7 @@ use crate::{
use anthropic_messages::AnthropicMessagesCapture;
pub(crate) use facts::ProviderFacts;
pub(crate) use facts::{InferenceFacts, InputOmission, InputOmissionReason, RelayOutcome};
+use openai_chat_completions::OpenAiChatCompletionsCapture;
use openai_responses::OpenAiResponsesCapture;
pub(crate) const MAX_INPUT_CAPTURE_BYTES: usize = 5 * 1024 * 1024;
@@ -27,6 +29,7 @@ const MAX_FACT_STRING: usize = 512;
enum ProtocolCapture {
OpenAiResponses(OpenAiResponsesCapture),
+ OpenAiChatCompletions(OpenAiChatCompletionsCapture),
AnthropicMessages(AnthropicMessagesCapture),
}
@@ -36,6 +39,9 @@ impl ProtocolCapture {
ApiFormat::OpenAiResponses => {
Self::OpenAiResponses(OpenAiResponsesCapture::new(headers, body, mode))
}
+ ApiFormat::OpenAiChatCompletions => {
+ Self::OpenAiChatCompletions(OpenAiChatCompletionsCapture::new(headers, body, mode))
+ }
ApiFormat::AnthropicMessages => {
Self::AnthropicMessages(AnthropicMessagesCapture::new(headers, body, mode))
}
@@ -45,6 +51,7 @@ impl ProtocolCapture {
fn record_response(&mut self, headers: &HeaderMap) {
match self {
Self::OpenAiResponses(capture) => capture.record_response(headers),
+ Self::OpenAiChatCompletions(capture) => capture.record_response(headers),
Self::AnthropicMessages(capture) => capture.record_response(headers),
}
}
@@ -52,6 +59,7 @@ impl ProtocolCapture {
fn push_bytes(&mut self, bytes: &[u8]) -> bool {
match self {
Self::OpenAiResponses(capture) => capture.push_bytes(bytes),
+ Self::OpenAiChatCompletions(capture) => capture.push_bytes(bytes),
Self::AnthropicMessages(capture) => capture.push_bytes(bytes),
}
}
@@ -59,6 +67,7 @@ impl ProtocolCapture {
fn end_body(&mut self) {
match self {
Self::OpenAiResponses(capture) => capture.end_body(),
+ Self::OpenAiChatCompletions(capture) => capture.end_body(),
Self::AnthropicMessages(capture) => capture.end_body(),
}
}
@@ -66,6 +75,9 @@ impl ProtocolCapture {
fn into_facts(self) -> (&'static str, ProviderFacts) {
match self {
Self::OpenAiResponses(capture) => ("openai.responses", capture.into_facts()),
+ Self::OpenAiChatCompletions(capture) => {
+ ("openai.chat-completions", capture.into_facts())
+ }
Self::AnthropicMessages(capture) => ("anthropic.messages", capture.into_facts()),
}
}
@@ -73,7 +85,7 @@ impl ProtocolCapture {
fn client_metadata(&self) -> Option<&Map<String, Value>> {
match self {
Self::OpenAiResponses(capture) => capture.client_metadata(),
- Self::AnthropicMessages(_) => None,
+ Self::OpenAiChatCompletions(_) | Self::AnthropicMessages(_) => None,
}
}
}
diff --git a/ai-gateway/src/capture/openai_chat_completions.rs b/ai-gateway/src/capture/openai_chat_completions.rs
new file mode 100644
index 0000000000..bb29946883
--- /dev/null
+++ b/ai-gateway/src/capture/openai_chat_completions.rs
@@ -0,0 +1,510 @@
+//! `OpenAI` Chat Completions request, JSON response and streamed choice capture.
+use super::{
+ MAX_FACT_STRING, MAX_ITEMS, MAX_OUTPUT_CAPTURE_BYTES, bounded_string, facts::ProviderFacts,
+ parse_request, response::ResponseBody,
+};
+use crate::resolution::IngestionMode;
+use axum::http::HeaderMap;
+use serde_json::{Map, Value, json};
+use std::collections::BTreeMap;
+
+const SCALAR_PARAMETERS: [&str; 17] = [
+ "temperature",
+ "top_p",
+ "n",
+ "max_tokens",
+ "max_completion_tokens",
+ "presence_penalty",
+ "frequency_penalty",
+ "seed",
+ "logprobs",
+ "top_logprobs",
+ "service_tier",
+ "parallel_tool_calls",
+ "reasoning_effort",
+ "verbosity",
+ "stream",
+ "store",
+ "prompt_cache_retention",
+];
+/// Structured configuration projected out of the recorded input in full mode.
+const STRUCTURED_PARAMETERS: [&str; 10] = [
+ "stop",
+ "response_format",
+ "tool_choice",
+ "function_call",
+ "stream_options",
+ "logit_bias",
+ "modalities",
+ "audio",
+ "prediction",
+ "web_search_options",
+];
+const REQUEST_METADATA: [&str; 4] = ["metadata", "prompt_cache_key", "safety_identifier", "user"];
+/// Streamed fields that identify rather than extend; repeated values replace.
+const IDENTITY_FIELDS: [&str; 3] = ["role", "id", "type"];
+
+pub(super) struct OpenAiChatCompletionsCapture {
+ facts: ProviderFacts,
+ mode: IngestionMode,
+ body: ResponseBody,
+ /// Present only in full mode; usage mode never records output.
+ output: Option<Choices>,
+ terminal: bool,
+ request_complete: bool,
+ response_valid: bool,
+}
+
+/// A capture gap: the recorded output no longer matches what was relayed.
+struct Gap;
+
+/// Choices rebuilt in full mode. Streamed choices are merged from their deltas;
+/// a choice that cannot be rebuilt exactly is dropped rather than recorded wrong.
+#[derive(Default)]
+struct Choices {
+ choices: BTreeMap<u64, Choice>,
+ bytes: usize,
+}
+
+#[derive(Default)]
+struct Choice {
+ fields: Map<String, Value>,
+ message: Map<String, Value>,
+ tool_calls: BTreeMap<u64, Map<String, Value>>,
+ /// This choice's share of the retained-output budget.
+ bytes: usize,
+ broken: bool,
+}
+
+impl Choices {
+ fn apply_chunk(&mut self, choice: &Map<String, Value>) -> Result<(), Gap> {
+ let index = choice.get("index").and_then(Value::as_u64).ok_or(Gap)?;
+ if !self.choices.contains_key(&index) && self.choices.len() >= MAX_ITEMS {
+ return Err(Gap);
+ }
+ let entry = self.choices.entry(index).or_default();
+ if entry.broken {
+ return Err(Gap);
+ }
+ let before = self.bytes;
+ let merged = Self::merge_choice(entry, choice, &mut self.bytes);
+ entry.bytes += self.bytes - before;
+ if merged.is_err() {
+ // A dropped choice is not recorded, so it releases its budget to its siblings.
+ self.bytes -= entry.bytes;
+ *entry = Choice {
+ broken: true,
+ ..Choice::default()
+ };
+ }
+ merged
+ }
+
+ fn merge_choice(
+ choice: &mut Choice,
+ source: &Map<String, Value>,
+ bytes: &mut usize,
+ ) -> Result<(), Gap> {
+ for (key, value) in source {
+ match key.as_str() {
+ "index" => {}
+ "delta" => {
+ for (key, value) in value.as_object().ok_or(Gap)? {
+ if key == "tool_calls" {
+ Self::merge_tool_calls(choice, value, bytes)?;
+ } else {
+ merge_field(&mut choice.message, key, value, bytes)?;
+ }
+ }
+ }
+ "finish_reason" if !value.is_null() => {
+ reserve(bytes, value.to_string().len())?;
+ choice.fields.insert(key.clone(), value.clone());
+ }
+ _ => merge_field(&mut choice.fields, key, value, bytes)?,
+ }
+ }
+ Ok(())
+ }
+
+ fn merge_tool_calls(choice: &mut Choice, calls: &Value, bytes: &mut usize) -> Result<(), Gap> {
+ for call in calls.as_array().ok_or(Gap)? {
+ let call = call.as_object().ok_or(Gap)?;
+ let index = call.get("index").and_then(Value::as_u64).ok_or(Gap)?;
+ if !choice.tool_calls.contains_key(&index) && choice.tool_calls.len() >= MAX_ITEMS {
+ return Err(Gap);
+ }
+ let target = choice.tool_calls.entry(index).or_default();
+ for (key, value) in call {
+ if key != "index" {
+ merge_field(target, key, value, bytes)?;
+ }
+ }
+ }
+ Ok(())
+ }
+
+ /// A non-streamed response carries its complete choices at once.
+ fn record_choices(&mut self, choices: &[Value]) -> Result<(), Gap> {
+ if choices.len() > MAX_ITEMS {
+ return Err(Gap);
+ }
+ for (position, choice) in choices.iter().enumerate() {
+ let fields = choice.as_object().ok_or(Gap)?.clone();
+ let index = fields
+ .get("index")
+ .and_then(Value::as_u64)
+ .unwrap_or(position as u64);
+ self.choices.insert(
+ index,
+ Choice {
+ fields,
+ ..Choice::default()
+ },
+ );
+ }
+ Ok(())
+ }
+
+ fn is_complete(&self) -> bool {
+ self.choices
+ .values()
+ .all(|choice| !choice.broken && choice.fields.contains_key("finish_reason"))
+ }
+
+ fn into_value(self) -> Value {
+ let choices = self
+ .choices
+ .into_iter()
+ .filter(|(_, choice)| !choice.broken)
+ .map(|(index, mut choice)| {
+ // A JSON choice is already native; a streamed one gains its message.
+ if !choice.message.is_empty() || !choice.tool_calls.is_empty() {
+ if !choice.tool_calls.is_empty() {
+ choice.message.insert(
+ "tool_calls".to_owned(),
+ Value::Array(
+ choice.tool_calls.into_values().map(Value::Object).collect(),
+ ),
+ );
+ }
+ choice
+ .fields
+ .insert("message".to_owned(), Value::Object(choice.message));
+ }
+ choice.fields.entry("index").or_insert(json!(index));
+ Value::Object(choice.fields)
+ })
+ .collect();
+ json!({ "choices": Value::Array(choices) })
+ }
+}
+
+/// Streamed text fragments append, arrays extend and objects merge field by
+/// field; identity fields and scalars replace. Nulls carry no fragment.
+fn merge_field(
+ target: &mut Map<String, Value>,
+ key: &str,
+ value: &Value,
+ bytes: &mut usize,
+) -> Result<(), Gap> {
+ match (target.get_mut(key), value) {
+ (_, Value::Null) => Ok(()),
+ (Some(Value::String(current)), Value::String(fragment))
+ if !IDENTITY_FIELDS.contains(&key) =>
+ {
+ reserve(bytes, fragment.len())?;
+ current.push_str(fragment);
+ Ok(())
+ }
+ (Some(Value::Array(current)), Value::Array(items)) => {
+ reserve(bytes, value.to_string().len())?;
+ current.extend(items.iter().cloned());
+ Ok(())
+ }
+ (Some(Value::Object(current)), Value::Object(fields)) => {
+ for (key, value) in fields {
+ merge_field(current, key, value, bytes)?;
+ }
+ Ok(())
+ }
+ (Some(current), _) if current.is_object() || current.is_array() => Err(Gap),
+ (_, value) => {
+ reserve(bytes, value.to_string().len())?;
+ target.insert(key.to_owned(), value.clone());
+ Ok(())
+ }
+ }
+}
+
+fn reserve(used: &mut usize, bytes: usize) -> Result<(), Gap> {
+ match used.checked_add(bytes) {
+ Some(total) if total <= MAX_OUTPUT_CAPTURE_BYTES => {
+ *used = total;
+ Ok(())
+ }
+ _ => Err(Gap),
+ }
+}
+
+/// The first generated text, refusal, tool or function arguments, or audio.
+fn has_generated_content(delta: &Map<String, Value>) -> bool {
+ let nonempty = |value: Option<&Value>| {
+ value
+ .and_then(Value::as_str)
+ .is_some_and(|value| !value.is_empty())
+ };
+ let arguments = |call: &Value| nonempty(call.pointer("/function/arguments"));
+ nonempty(delta.get("content"))
+ || nonempty(delta.get("refusal"))
+ || delta
+ .get("tool_calls")
+ .and_then(Value::as_array)
+ .is_some_and(|calls| calls.iter().any(arguments))
+ || nonempty(
+ delta
+ .get("function_call")
+ .and_then(|call| call.get("arguments")),
+ )
+ || delta
+ .get("audio")
+ .is_some_and(|audio| nonempty(audio.get("data")) || nonempty(audio.get("transcript")))
+}
+
+impl OpenAiChatCompletionsCapture {
+ pub fn new(headers: &HeaderMap, body: &[u8], mode: IngestionMode) -> Self {
+ let mut capture = Self {
+ facts: ProviderFacts::default(),
+ mode,
+ body: ResponseBody::Unknown,
+ output: (mode == IngestionMode::Full).then(Choices::default),
+ terminal: false,
+ request_complete: false,
+ response_valid: true,
+ };
+ match parse_request(headers, body) {
+ Ok(request) => {
+ capture.capture_request(request);
+ capture.request_complete = true;
+ }
+ Err(omission) if mode == IngestionMode::Full => {
+ capture.facts.input_omission = Some(omission);
+ }
+ Err(_) => {}
+ }
+ capture
+ }
+
+ pub fn record_response(&mut self, headers: &HeaderMap) {
+ self.facts.provider_request_id = headers
+ .get("x-request-id")
+ .and_then(|v| v.to_str().ok())
+ .and_then(bounded_string);
+ self.body = ResponseBody::from_headers(headers);
+ if matches!(self.body, ResponseBody::Unavailable) {
+ self.response_valid = false;
+ }
+ }
+
+ pub fn push_bytes(&mut self, bytes: &[u8]) -> bool {
+ let mut body = std::mem::replace(&mut self.body, ResponseBody::Unavailable);
+ let mut completion_started = false;
+ if !body.push(bytes, |event| {
+ completion_started |= self.handle_event(event);
+ }) {
+ self.response_valid = false;
+ }
+ self.body = body;
+ completion_started
+ }
+
+ pub fn end_body(&mut self) {
+ match std::mem::replace(&mut self.body, ResponseBody::Unavailable) {
+ ResponseBody::Json(bytes) => match serde_json::from_slice::<Value>(&bytes) {
+ Ok(Value::Object(response)) => {
+ self.capture_chunk_facts(&response);
+ if let Some(choices) = response.get("choices").and_then(Value::as_array) {
+ self.capture_json_choices(choices);
+ self.terminal = true;
+ }
+ }
+ _ => self.response_valid = false,
+ },
+ ResponseBody::Sse(sse) => {
+ self.response_valid &= sse.finish();
+ }
+ ResponseBody::Unknown | ResponseBody::Unavailable => {}
+ }
+ self.facts.output_complete = self.terminal
+ && self.response_valid
+ && self.output.as_ref().is_none_or(Choices::is_complete);
+ self.facts.capture_complete = self.request_complete && self.facts.output_complete;
+ }
+
+ pub fn into_facts(mut self) -> ProviderFacts {
+ if matches!(self.body, ResponseBody::Sse(_)) {
+ self.end_body();
+ }
+ self.facts.output = self.output.take().map(Choices::into_value);
+ self.facts
+ }
+
+ fn capture_request(&mut self, mut request: Map<String, Value>) {
+ self.facts.requested_model = request
+ .remove("model")
+ .as_ref()
+ .and_then(Value::as_str)
+ .and_then(bounded_string);
+ self.facts.model.clone_from(&self.facts.requested_model);
+ for key in SCALAR_PARAMETERS {
+ if let Some(value) = request.remove(key).filter(|v| {
+ self.mode == IngestionMode::Full
+ || v.is_number()
+ || v.is_boolean()
+ || v.as_str().is_some_and(|s| s.len() <= MAX_FACT_STRING)
+ }) {
+ self.facts.model_parameters.insert(key.to_owned(), value);
+ }
+ }
+ if self.mode == IngestionMode::Full {
+ for key in STRUCTURED_PARAMETERS {
+ if let Some(value) = request.remove(key) {
+ self.facts.model_parameters.insert(key.to_owned(), value);
+ }
+ }
+ for key in REQUEST_METADATA {
+ if let Some(value) = request.remove(key) {
+ self.facts.request_metadata.insert(key.to_owned(), value);
+ }
+ }
+ // `messages`, `tools`, legacy `functions` and unknown fields remain native.
+ self.facts.input = Some(Value::Object(request));
+ self.facts.input_complete = true;
+ }
+ }
+
+ fn capture_json_choices(&mut self, choices: &[Value]) {
+ for choice in choices {
+ if let Some(reason) = choice.get("finish_reason") {
+ self.capture_finish_reason(reason);
+ }
+ }
+ if self
+ .output
+ .as_mut()
+ .is_some_and(|output| output.record_choices(choices).is_err())
+ {
+ self.response_valid = false;
+ }
+ }
+
+ fn handle_event(&mut self, bytes: &[u8]) -> bool {
+ if bytes == b"[DONE]" {
+ self.terminal = true;
+ return false;
+ }
+ let Ok(Value::Object(chunk)) = serde_json::from_slice(bytes) else {
+ self.response_valid = false;
+ return false;
+ };
+ self.capture_chunk_facts(&chunk);
+ let Some(choices) = chunk.get("choices") else {
+ return false;
+ };
+ let Some(choices) = choices.as_array() else {
+ self.response_valid = false;
+ return false;
+ };
+ let mut completion_started = false;
+ for choice in choices {
+ let Some(choice) = choice.as_object() else {
+ self.response_valid = false;
+ continue;
+ };
+ if let Some(reason) = choice.get("finish_reason") {
+ self.capture_finish_reason(reason);
+ }
+ completion_started |= choice
+ .get("delta")
+ .and_then(Value::as_object)
+ .is_some_and(has_generated_content);
+ if self
+ .output
+ .as_mut()
+ .is_some_and(|output| output.apply_chunk(choice).is_err())
+ {
+ self.response_valid = false;
+ }
+ }
+ completion_started
+ }
+
+ /// Facts every chunk and the JSON response carry at their top level.
+ fn capture_chunk_facts(&mut self, chunk: &Map<String, Value>) {
+ if let Some(error) = chunk.get("error").and_then(Value::as_object) {
+ self.facts.provider_status = Some("failed".to_owned());
+ self.record_error(error);
+ }
+ for (key, target) in [
+ ("id", &mut self.facts.provider_response_id),
+ ("model", &mut self.facts.model),
+ ] {
+ if let Some(value) = chunk
+ .get(key)
+ .and_then(Value::as_str)
+ .and_then(bounded_string)
+ {
+ *target = Some(value);
+ }
+ }
+ if let Some(tier) = chunk
+ .get("service_tier")
+ .and_then(Value::as_str)
+ .and_then(bounded_string)
+ {
+ self.facts
+ .model_parameters
+ .insert("service_tier".to_owned(), Value::String(tier));
+ }
+ if let Some(usage) = chunk.get("usage").filter(|v| v.is_object()) {
+ // The enclosing JSON body or SSE event already enforces the byte limit.
+ self.facts.usage_details = Some(usage.clone());
+ }
+ }
+
+ /// A truncated or filtered choice makes the response incomplete; failure wins.
+ fn capture_finish_reason(&mut self, reason: &Value) {
+ let Some(reason) = reason.as_str() else {
+ return;
+ };
+ let status = match reason {
+ "length" | "content_filter" => "incomplete",
+ _ => "completed",
+ };
+ match self.facts.provider_status.as_deref() {
+ Some("failed" | "incomplete") => {}
+ _ => self.facts.provider_status = Some(status.to_owned()),
+ }
+ }
+
+ fn record_error(&mut self, error: &Map<String, Value>) {
+ // Provider error messages may echo prompt content, so retain them only in full mode.
+ if self.mode != IngestionMode::Full {
+ return;
+ }
+ let detail = ["code", "message"]
+ .into_iter()
+ .filter_map(|key| error.get(key).and_then(Value::as_str))
+ .filter(|value| !value.is_empty())
+ .map(|value| &value[..value.floor_char_boundary(MAX_FACT_STRING)])
+ .collect::<Vec<_>>()
+ .join(": ");
+ if !detail.is_empty() {
+ self.facts.error_message =
+ Some(detail[..detail.floor_char_boundary(MAX_FACT_STRING)].to_owned());
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/ai-gateway/src/capture/openai_chat_completions/tests.rs b/ai-gateway/src/capture/openai_chat_completions/tests.rs
new file mode 100644
index 0000000000..ac9e7d8352
--- /dev/null
+++ b/ai-gateway/src/capture/openai_chat_completions/tests.rs
@@ -0,0 +1,402 @@
+use super::*;
+use crate::{
+ capture::{
+ ExecutionCapture, InputOmission, InputOmissionReason, MAX_INPUT_CAPTURE_BYTES,
+ ProtocolCapture, ProviderFacts,
+ },
+ resolution::ApiFormat,
+ test_support::resolved_request_context_for,
+};
+use axum::http::{HeaderValue, header};
+use serde_json::json;
+
+const REQUEST: &[u8] = br#"{"model":"gpt-4.1-mini","messages":[{"role":"system","content":"system-canary"},{"role":"user","content":"prompt-canary"}],"tools":[{"type":"function","function":{"name":"lookup","parameters":{"type":"object"}}}],"temperature":0.2,"n":1,"max_completion_tokens":256,"stream":true,"stop":["stop-canary"],"response_format":{"type":"text"},"tool_choice":"auto","stream_options":{"include_usage":true},"metadata":{"customer":"customer-1"},"user":"legacy-user","future_field":{"kept":true}}"#;
+
+fn captured(execution: &ExecutionCapture) -> &OpenAiChatCompletionsCapture {
+ let ProtocolCapture::OpenAiChatCompletions(capture) = execution.protocol.as_ref().unwrap()
+ else {
+ panic!("expected the Chat Completions adapter");
+ };
+ capture
+}
+
+async fn new_observer(mode: &'static str) -> ExecutionCapture {
+ new_observer_for(mode, REQUEST).await
+}
+
+async fn new_observer_for(mode: &'static str, request: &[u8]) -> ExecutionCapture {
+ let context =
+ resolved_request_context_for(ApiFormat::OpenAiChatCompletions, "provider-secret", mode)
+ .await;
+ ExecutionCapture::for_request(
+ ApiFormat::OpenAiChatCompletions,
+ &context,
+ &HeaderMap::new(),
+ request,
+ )
+}
+
+fn record_response(observer: &mut ExecutionCapture, status: u16, content_type: &'static str) {
+ let mut headers = HeaderMap::new();
+ headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type));
+ headers.insert("x-request-id", HeaderValue::from_static("req_provider_1"));
+ observer.record_response(status, &headers);
+}
+
+fn finalized(observer: &mut ExecutionCapture) -> (&'static str, ProviderFacts) {
+ observer.protocol.take().unwrap().into_facts()
+}
+
+fn chunk(choices: Value) -> String {
+ let mut value = json!({
+ "id": "chatcmpl-1", "object": "chat.completion.chunk", "created": 1,
+ "model": "gpt-4.1-mini-2025-04-14", "service_tier": "default",
+ "system_fingerprint": "fp_1", "usage": null,
+ });
+ value["choices"] = choices;
+ format!("data: {value}\n\n")
+}
+
+fn delta(index: u64, delta: Value, finish_reason: Value) -> String {
+ let mut choice = json!({"index": index, "logprobs": null});
+ choice["delta"] = delta;
+ choice["finish_reason"] = finish_reason;
+ chunk(Value::Array(vec![choice]))
+}
+
+const USAGE: &str = "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4.1-mini-2025-04-14\",\"choices\":[],\"usage\":{\"prompt_tokens\":19,\"completion_tokens\":10,\"total_tokens\":29,\"prompt_tokens_details\":{\"cached_tokens\":0,\"audio_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":0}}}\n\n";
+const DONE: &str = "data: [DONE]\n\n";
+
+fn text_stream() -> String {
+ [
+ delta(
+ 0,
+ json!({"role":"assistant","content":"","refusal":null}),
+ Value::Null,
+ ),
+ delta(0, json!({"content":"Hel"}), Value::Null), // codespell:ignore
+ delta(0, json!({"content":"lo"}), Value::Null),
+ delta(0, json!({}), json!("stop")),
+ USAGE.to_owned(),
+ DONE.to_owned(),
+ ]
+ .concat()
+}
+
+#[tokio::test]
+async fn streamed_text_and_tool_calls_are_rebuilt_into_native_choices() {
+ let mut observer = new_observer("full").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ let stream = [
+ delta(0, json!({"role":"assistant","content":null,"refusal":null}), Value::Null),
+ delta(0, json!({"content":"Checking"}), Value::Null),
+ delta(
+ 0,
+ json!({"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"lookup","arguments":""}}]}),
+ Value::Null,
+ ),
+ delta(
+ 0,
+ json!({"tool_calls":[{"index":0,"function":{"arguments":"{\"q\":"}}]}),
+ Value::Null,
+ ),
+ delta(
+ 0,
+ json!({"tool_calls":[{"index":1,"id":"call_2","type":"function","function":{"name":"lookup","arguments":"{}"}}]}),
+ Value::Null,
+ ),
+ delta(
+ 0,
+ json!({"tool_calls":[{"index":0,"function":{"arguments":"\"x\"}"}}]}),
+ Value::Null,
+ ),
+ delta(0, json!({}), json!("tool_calls")),
+ USAGE.to_owned(),
+ DONE.to_owned(),
+ ]
+ .concat();
+ // Arbitrary byte splits must not change the capture.
+ for piece in stream.as_bytes().chunks(7) {
+ observer.push_bytes(piece);
+ }
+ observer.end_body();
+ let capture = captured(&observer);
+ assert!(capture.facts.output_complete);
+ assert!(capture.facts.capture_complete);
+ assert_eq!(capture.facts.provider_status.as_deref(), Some("completed"));
+ let (api_format, facts) = finalized(&mut observer);
+ assert_eq!(api_format, "openai.chat-completions");
+ assert_eq!(
+ facts.output,
+ Some(json!({"choices": [{
+ "index": 0,
+ "finish_reason": "tool_calls",
+ "message": {
+ "role": "assistant",
+ "content": "Checking",
+ "tool_calls": [
+ {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{\"q\":\"x\"}"}},
+ {"id": "call_2", "type": "function", "function": {"name": "lookup", "arguments": "{}"}},
+ ],
+ },
+ }]}))
+ );
+ assert_eq!(facts.provider_response_id.as_deref(), Some("chatcmpl-1"));
+ assert_eq!(facts.provider_request_id.as_deref(), Some("req_provider_1"));
+ assert_eq!(facts.model.as_deref(), Some("gpt-4.1-mini-2025-04-14"));
+ assert_eq!(facts.requested_model.as_deref(), Some("gpt-4.1-mini"));
+ assert_eq!(facts.model_parameters["service_tier"], "default");
+ assert_eq!(facts.usage_details.unwrap()["total_tokens"], 29);
+}
+
+#[tokio::test]
+async fn json_and_streaming_capture_equivalent_choices_and_facts() {
+ const JSON: &[u8] = br#"{"id":"chatcmpl-1","object":"chat.completion","created":1,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","choices":[{"index":0,"message":{"role":"assistant","content":"Hello"},"finish_reason":"stop"}],"usage":{"prompt_tokens":19,"completion_tokens":10,"total_tokens":29,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}}"#;
+ let mut json_observer = new_observer("full").await;
+ record_response(&mut json_observer, 200, "application/json");
+ json_observer.push_bytes(JSON);
+ json_observer.end_body();
+ assert!(json_observer.completion_start_ms.is_none());
+ assert!(captured(&json_observer).facts.output_complete);
+
+ let mut sse_observer = new_observer("full").await;
+ record_response(&mut sse_observer, 200, "text/event-stream");
+ sse_observer.push_bytes(text_stream().as_bytes());
+ sse_observer.end_body();
+ assert!(sse_observer.completion_start_ms.is_some());
+
+ let (_, json) = finalized(&mut json_observer);
+ let (_, sse) = finalized(&mut sse_observer);
+ assert_eq!(
+ json.output,
+ Some(
+ json!({"choices":[{"index":0,"message":{"role":"assistant","content":"Hello"},"finish_reason":"stop"}]})
+ )
+ );
+ assert_eq!(sse.output, json.output);
+ assert_eq!(sse.usage_details, json.usage_details);
+ assert_eq!(sse.model, json.model);
+ assert_eq!(sse.provider_response_id, json.provider_response_id);
+ assert_eq!(sse.provider_status.as_deref(), Some("completed"));
+ assert_eq!(json.provider_status.as_deref(), Some("completed"));
+}
+
+#[tokio::test]
+async fn request_configuration_and_metadata_are_projected_out_of_input() {
+ let mut observer = new_observer("full").await;
+ let (_, facts) = finalized(&mut observer);
+ assert_eq!(
+ facts.input,
+ Some(json!({
+ "messages": [{"role":"system","content":"system-canary"},{"role":"user","content":"prompt-canary"}],
+ "tools": [{"type":"function","function":{"name":"lookup","parameters":{"type":"object"}}}],
+ "future_field": {"kept": true},
+ }))
+ );
+ assert!(facts.input_complete);
+ assert_eq!(
+ Value::Object(facts.model_parameters),
+ json!({
+ "temperature": 0.2, "n": 1, "max_completion_tokens": 256, "stream": true,
+ "stop": ["stop-canary"], "response_format": {"type":"text"}, "tool_choice": "auto",
+ "stream_options": {"include_usage": true},
+ })
+ );
+ assert_eq!(
+ Value::Object(facts.request_metadata),
+ json!({"metadata": {"customer": "customer-1"}, "user": "legacy-user"})
+ );
+}
+
+#[tokio::test]
+async fn usage_mode_keeps_scalars_and_usage_but_no_content() {
+ let mut observer = new_observer("usage").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(text_stream().as_bytes());
+ observer.end_body();
+ assert!(observer.completion_start_ms.is_some());
+ assert!(captured(&observer).facts.output_complete);
+ let (_, facts) = finalized(&mut observer);
+ assert!(facts.input.is_none());
+ assert!(facts.output.is_none());
+ assert!(facts.request_metadata.is_empty());
+ assert_eq!(
+ Value::Object(facts.model_parameters.clone()),
+ json!({"temperature": 0.2, "n": 1, "max_completion_tokens": 256, "stream": true, "service_tier": "default"})
+ );
+ let serialized = serde_json::to_string(&facts).unwrap();
+ for canary in [
+ "prompt-canary",
+ "system-canary",
+ "stop-canary",
+ "customer-1",
+ "provider-secret",
+ ] {
+ assert!(!serialized.contains(canary), "{canary}");
+ }
+ assert_eq!(facts.usage_details.unwrap()["prompt_tokens"], 19);
+}
+
+#[tokio::test]
+async fn completion_starts_on_generated_content_not_role_or_tool_names() {
+ for (delta_fields, expected) in [
+ (json!({"content":"x"}), true),
+ (json!({"refusal":"no"}), true),
+ (
+ json!({"tool_calls":[{"index":0,"function":{"arguments":"{"}}]}),
+ true,
+ ),
+ (json!({"function_call":{"arguments":"{"}}), true),
+ (json!({"audio":{"id":"audio_1","transcript":"hi"}}), true),
+ (json!({"role":"assistant","content":""}), false),
+ (
+ json!({"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"lookup","arguments":""}}]}),
+ false,
+ ),
+ ] {
+ let mut observer = new_observer("usage").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(chunk(json!([])).as_bytes());
+ assert!(observer.completion_start_ms.is_none());
+ observer.push_bytes(delta(0, delta_fields.clone(), Value::Null).as_bytes());
+ assert_eq!(
+ observer.completion_start_ms.is_some(),
+ expected,
+ "{delta_fields}"
+ );
+ }
+}
+
+#[tokio::test]
+async fn finish_reasons_errors_and_missing_done_set_status_and_completeness() {
+ for (reason, status) in [
+ ("length", "incomplete"),
+ ("content_filter", "incomplete"),
+ ("stop", "completed"),
+ ] {
+ let mut observer = new_observer("usage").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(delta(0, json!({"content":"x"}), json!(reason)).as_bytes());
+ observer.push_bytes(DONE.as_bytes());
+ observer.end_body();
+ assert_eq!(
+ captured(&observer).facts.provider_status.as_deref(),
+ Some(status)
+ );
+ assert!(captured(&observer).facts.output_complete);
+ }
+
+ // A stream that never reaches `[DONE]` keeps its facts but is not complete.
+ let mut observer = new_observer("full").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(delta(0, json!({"content":"partial"}), Value::Null).as_bytes());
+ observer.end_body();
+ assert!(!captured(&observer).facts.output_complete);
+ let (_, facts) = finalized(&mut observer);
+ assert_eq!(
+ facts.output.unwrap()["choices"][0]["message"]["content"],
+ "partial"
+ );
+ assert!(facts.usage_details.is_none());
+
+ // `[DONE]` with an unfinished choice is not a complete full-mode output.
+ let mut observer = new_observer("full").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(delta(0, json!({"content":"partial"}), Value::Null).as_bytes());
+ observer.push_bytes(DONE.as_bytes());
+ observer.end_body();
+ assert!(!captured(&observer).facts.output_complete);
+
+ for (mode, expected) in [
+ ("full", Some("server_error: prompt-canary echoed")),
+ ("usage", None),
+ ] {
+ let mut observer = new_observer(mode).await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(delta(0, json!({"content":"x"}), json!("stop")).as_bytes());
+ observer.push_bytes(
+ b"data: {\"error\":{\"message\":\"prompt-canary echoed\",\"type\":\"server_error\",\"code\":\"server_error\"}}\n\n",
+ );
+ observer.end_body();
+ let facts = &captured(&observer).facts;
+ assert_eq!(facts.provider_status.as_deref(), Some("failed"));
+ assert_eq!(facts.error_message.as_deref(), expected);
+
+ let mut observer = new_observer(mode).await;
+ record_response(&mut observer, 429, "application/json");
+ observer.push_bytes(br#"{"error":{"message":"prompt-canary echoed","type":"requests","code":"rate_limit_exceeded"}}"#);
+ observer.end_body();
+ let facts = &captured(&observer).facts;
+ assert_eq!(facts.provider_status.as_deref(), Some("failed"));
+ assert!(!facts.output_complete);
+ assert_eq!(
+ facts.error_message.is_some(),
+ mode == "full",
+ "{:?}",
+ facts.error_message
+ );
+ }
+}
+
+#[tokio::test]
+async fn multiple_choices_merge_independently_and_a_gap_drops_only_its_choice() {
+ let mut observer = new_observer("full").await;
+ record_response(&mut observer, 200, "text/event-stream");
+ observer.push_bytes(
+ chunk(json!([
+ {"index":0,"delta":{"role":"assistant","content":"A"},"finish_reason":null},
+ {"index":1,"delta":{"role":"assistant","content":"B"},"finish_reason":null},
+ ]))
+ .as_bytes(),
+ );
+ // Fragments past the retained-output budget cannot be recorded exactly.
+ let fragment = "x".repeat(MAX_OUTPUT_CAPTURE_BYTES * 2 / 5);
+ for _ in 0..3 {
+ observer.push_bytes(delta(1, json!({ "content": fragment }), Value::Null).as_bytes());
+ }
+ observer.push_bytes(delta(1, json!({"content":"!"}), json!("stop")).as_bytes());
+ // The dropped choice's budget returns to its siblings; this only fits after that.
+ for _ in 0..2 {
+ observer.push_bytes(delta(0, json!({ "content": fragment }), Value::Null).as_bytes());
+ }
+ observer.push_bytes(delta(0, json!({"content":"a"}), json!("stop")).as_bytes());
+ observer.push_bytes(DONE.as_bytes());
+ observer.end_body();
+ assert!(!captured(&observer).facts.output_complete);
+ let (_, facts) = finalized(&mut observer);
+ assert_eq!(
+ facts.output,
+ Some(json!({"choices":[{
+ "index":0,"finish_reason":"stop",
+ "message":{"role":"assistant","content":format!("A{fragment}{fragment}a")},
+ }]}))
+ );
+}
+
+#[tokio::test]
+async fn oversized_encoded_and_invalid_requests_explain_the_missing_input() {
+ let mut large = br#"{"model":"gpt-4.1-mini","messages":[{"role":"user","content":""#.to_vec();
+ large.resize(MAX_INPUT_CAPTURE_BYTES + 1 - br#""}]}"#.len(), b'x');
+ large.extend_from_slice(br#""}]}"#);
+ let mut observer = new_observer_for("full", &large).await;
+ let (_, facts) = finalized(&mut observer);
+ assert_eq!(
+ facts.input_omission,
+ Some(InputOmission {
+ reason: InputOmissionReason::SizeLimit,
+ body_bytes: MAX_INPUT_CAPTURE_BYTES + 1,
+ })
+ );
+ assert!(facts.input.is_none());
+ assert!(facts.requested_model.is_none());
+
+ let mut observer = new_observer_for("full", b"not json").await;
+ let (_, facts) = finalized(&mut observer);
+ assert_eq!(
+ facts.input_omission.map(|omission| omission.reason),
+ Some(InputOmissionReason::InvalidJson)
+ );
+
+ let mut observer = new_observer_for("usage", b"not json").await;
+ assert!(finalized(&mut observer).1.input_omission.is_none());
+}
diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rs
index 9087ad117e..f1b803a58b 100644
--- a/ai-gateway/src/http.rs
+++ b/ai-gateway/src/http.rs
@@ -37,6 +37,7 @@ pub fn router(inference: Option<InferenceService>, lifecycle: GatewayLifecycleSt
"/openai/v1/responses/compact",
post(handle_responses_compact),
)
+ .route("/openai/v1/chat/completions", post(handle_chat_completions))
.route("/openai/v1/models", get(handle_openai_models))
.route("/anthropic/v1/messages", post(handle_messages))
.route(
@@ -62,6 +63,13 @@ async fn handle_responses_compact(
handle(state, request, Route::OpenAiResponsesCompact).await
}
+async fn handle_chat_completions(
+ State(state): State<InferenceRouteState>,
+ request: Request,
+) -> Response {
+ handle(state, request, Route::OpenAiChatCompletions).await
+}
+
async fn handle_openai_models(
State(state): State<InferenceRouteState>,
request: Request,
@@ -187,7 +195,7 @@ fn gateway_key(headers: &HeaderMap, api_format: ApiFormat) -> Result<&str, Infer
})
.transpose()?;
let api_key = match api_format {
- ApiFormat::OpenAiResponses => None,
+ ApiFormat::OpenAiResponses | ApiFormat::OpenAiChatCompletions => None,
ApiFormat::AnthropicMessages => single_header(headers, "x-api-key")?,
};
let key = match (api_key, bearer) {
@@ -388,7 +396,7 @@ impl InferenceHttpError {
);
}
match api_format {
- ApiFormat::OpenAiResponses => (
+ ApiFormat::OpenAiResponses | ApiFormat::OpenAiChatCompletions => (
status,
Json(OpenAiErrorResponse {
error: OpenAiErrorDetail {
diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rs
index 533f15353c..ed83c74f62 100644
--- a/ai-gateway/src/http/tests.rs
+++ b/ai-gateway/src/http/tests.rs
@@ -177,6 +177,10 @@ impl Gateway {
reqwest::Client::new().post(format!("{}/openai/v1/responses", self.url))
}
+ fn chat_completions(&self) -> reqwest::RequestBuilder {
+ reqwest::Client::new().post(format!("{}/openai/v1/chat/completions", self.url))
+ }
+
fn compact(&self) -> reqwest::RequestBuilder {
reqwest::Client::new().post(format!("{}/openai/v1/responses/compact", self.url))
}
@@ -218,7 +222,13 @@ async fn every_error_carries_a_fresh_request_id_in_its_header_and_native_body()
if request.headers().get(header::AUTHORIZATION)
== Some(&header::HeaderValue::from_static("Bearer valid"))
{
- return resolution_response("provider-secret");
+ let body = to_bytes(request.into_body(), 1024).await.unwrap();
+ let api_format =
+ serde_json::from_slice::<serde_json::Value>(&body).unwrap()["apiFormat"].clone();
+ return resolution_response_for(
+ serde_json::from_value(api_format).unwrap(),
+ "provider-secret",
+ );
}
Response::builder()
.status(StatusCode::UNAUTHORIZED)
@@ -238,6 +248,24 @@ async fn every_error_carries_a_fresh_request_id_in_its_header_and_native_body()
true,
),
(configured.post(), StatusCode::UNAUTHORIZED, false),
+ (
+ unconfigured.chat_completions(),
+ StatusCode::SERVICE_UNAVAILABLE,
+ false,
+ ),
+ (
+ configured.chat_completions().bearer_auth("rejected"),
+ StatusCode::UNAUTHORIZED,
+ false,
+ ),
+ (
+ configured
+ .chat_completions()
+ .bearer_auth("valid")
+ .body(vec![b'x'; MAX_REQUEST_BYTES + 1]),
+ StatusCode::PAYLOAD_TOO_LARGE,
+ false,
+ ),
(
configured.post().bearer_auth("rejected"),
StatusCode::UNAUTHORIZED,
@@ -687,6 +715,112 @@ async fn pre_header_provider_failures_export_the_status_returned_to_the_caller()
}
}
+#[tokio::test]
+async fn chat_completions_streams_carry_the_request_and_generation_ids() {
+ use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};
+ use serde_json::Value;
+
+ const CONTENT: &str = "data: {\"id\":\"chatcmpl-1\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"hi\"},\"finish_reason\":\"stop\"}],\"usage\":null}\n\n";
+ const USAGE: &str = "data: {\"id\":\"chatcmpl-1\",\"choices\":[],\"usage\":{\"prompt_tokens\":2,\"completion_tokens\":1,\"total_tokens\":3}}\n\n";
+ const DONE: &str = "data: [DONE]\n\n";
+ let (sent, mut received) = tokio::sync::mpsc::channel(1);
+ let sink = FakeServer::start(move |request| {
+ let sent = sent.clone();
+ async move {
+ let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
+ sent.send(crate::test_support::upload_json(&bytes))
+ .await
+ .unwrap();
+ Response::new(Body::from("{}"))
+ }
+ })
+ .await;
+ let telemetry =
+ Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
+ let (resolved, mut resolution_ids) = tokio::sync::mpsc::channel(1);
+ let web = FakeServer::start(move |request| {
+ let resolved = resolved.clone();
+ async move {
+ let id = request.headers()["langfuse-gateway-request-id"]
+ .to_str()
+ .unwrap()
+ .to_owned();
+ resolved.send(id).await.unwrap();
+ resolution_response_for(ApiFormat::OpenAiChatCompletions, "provider-chat")
+ }
+ })
+ .await;
+ let provider = FakeServer::start(|request| async move {
+ assert_eq!(request.uri(), "/v1/chat/completions");
+ Response::builder()
+ .header("content-type", "text/event-stream")
+ .header("x-request-id", "req-upstream")
+ .body(Body::from([CONTENT, USAGE, DONE].concat()))
+ .unwrap()
+ })
+ .await;
+ let gateway = Gateway::start(Some(InferenceService::for_test(
+ web.control_plane(),
+ ProviderTransport::for_test(format!("{}/v1", provider.url), ProviderLimits::default())
+ .with_telemetry(telemetry.clone()),
+ 128,
+ )))
+ .await;
+ let response = gateway
+ .chat_completions()
+ .bearer_auth("gateway-chat")
+ .header("x-request-id", "client-request")
+ .body(r#"{"model":"gpt-4.1-mini","messages":[],"stream":true}"#)
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::OK);
+ let id = request_id(&response);
+ let header = |name: &str| response.headers()[name].to_str().unwrap().to_owned();
+ let (trace_id, observation_id) = (
+ header("langfuse-trace-id"),
+ header("langfuse-observation-id"),
+ );
+ assert_eq!(header("x-request-id"), "req-upstream");
+ assert_eq!(
+ response.text().await.unwrap(),
+ [CONTENT, USAGE, DONE].concat()
+ );
+ assert_eq!(resolution_ids.recv().await.unwrap(), id);
+
+ let payload = tokio::time::timeout(Duration::from_secs(2), received.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];
+ assert_eq!(span["name"], "openai.chat-completions");
+ assert_eq!(span["traceId"], trace_id.as_str());
+ assert_eq!(span["spanId"], observation_id.as_str());
+ let metadata: Value = serde_json::from_str(
+ span["attributes"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|a| a["key"] == "langfuse.observation.metadata")
+ .unwrap()["value"]["stringValue"]
+ .as_str()
+ .unwrap(),
+ )
+ .unwrap();
+ assert_eq!(metadata["langfuse.gateway.request.id"], id.as_str());
+ assert_eq!(
+ metadata["langfuse.gateway.client.request.id"],
+ "client-request"
+ );
+ assert_eq!(
+ metadata["langfuse.gateway.upstream.request.id"],
+ "req-upstream"
+ );
+ telemetry
+ .shutdown(tokio::time::Instant::now() + Duration::from_secs(1))
+ .await;
+}
+
#[tokio::test]
async fn unconfigured_gateway_is_live_but_not_ready_and_inference_is_unavailable() {
let gateway = Gateway::start(None).await;
@@ -1043,20 +1177,84 @@ async fn openai_routes_ignore_x_api_key_and_keep_their_envelope() {
.await;
let provider = FakeServer::start(|_| async { Response::new(Body::empty()) }).await;
let gateway = Gateway::start(Some(inference(&web, &provider))).await;
- let response = gateway
- .post()
- .header("x-api-key", "gateway-key")
- .body("{}")
- .send()
- .await
- .unwrap();
- assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
- let body: serde_json::Value = serde_json::from_str(&response.text().await.unwrap()).unwrap();
- assert_eq!(body["error"]["code"], "invalid_api_key");
- assert!(body.get("type").is_none());
+ for request in [gateway.post(), gateway.chat_completions()] {
+ let response = request
+ .header("x-api-key", "gateway-key")
+ .body("{}")
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+ let body: serde_json::Value =
+ serde_json::from_str(&response.text().await.unwrap()).unwrap();
+ assert_eq!(body["error"]["code"], "invalid_api_key");
+ assert!(body.get("type").is_none());
+ }
assert_eq!(web.calls(), 0);
}
+#[tokio::test]
+async fn chat_completions_resolve_their_own_format_and_relay_native_json_and_errors() {
+ let web = FakeServer::start(|request| async move {
+ assert_eq!(
+ to_bytes(request.into_body(), 1024).await.unwrap(),
+ r#"{"apiFormat":"openai.chat-completions"}"#
+ );
+ resolution_response_for(ApiFormat::OpenAiChatCompletions, "provider-chat")
+ })
+ .await;
+ let provider = FakeServer::start(|request| async move {
+ assert_eq!(request.uri(), "/v1/chat/completions");
+ assert_eq!(
+ request.headers()[header::AUTHORIZATION],
+ "Bearer provider-chat"
+ );
+ let body = to_bytes(request.into_body(), 4096).await.unwrap();
+ if body == r#"{"model":"unknown-model","messages":[]}"# {
+ return Response::builder()
+ .status(StatusCode::NOT_FOUND)
+ .header("content-type", "application/json")
+ .body(Body::from(r#"{"error":{"code":"model_not_found"}}"#))
+ .unwrap();
+ }
+ assert_eq!(body, r#"{ "model":"gpt-4.1-mini", "messages":[] }"#);
+ Response::builder()
+ .header("content-type", "application/json")
+ .body(Body::from(r#"{"id":"chatcmpl-1","choices":[]}"#))
+ .unwrap()
+ })
+ .await;
+ let gateway = Gateway::start(Some(inference(&web, &provider))).await;
+ for (body, status, expected) in [
+ (
+ r#"{ "model":"gpt-4.1-mini", "messages":[] }"#,
+ StatusCode::OK,
+ r#"{"id":"chatcmpl-1","choices":[]}"#,
+ ),
+ (
+ r#"{"model":"unknown-model","messages":[]}"#,
+ StatusCode::NOT_FOUND,
+ r#"{"error":{"code":"model_not_found"}}"#,
+ ),
+ ] {
+ let response = gateway
+ .chat_completions()
+ .bearer_auth("gateway-chat")
+ .body(body)
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.status(), status);
+ assert_eq!(response.text().await.unwrap(), expected);
+ }
+ let missing = gateway.chat_completions().body("{}").send().await.unwrap();
+ assert_eq!(missing.status(), StatusCode::UNAUTHORIZED);
+ let body: serde_json::Value = serde_json::from_str(&missing.text().await.unwrap()).unwrap();
+ assert_eq!(body["error"]["type"], "authentication_error");
+ assert_eq!(web.calls(), 2);
+ assert_eq!(provider.calls(), 2);
+}
+
#[tokio::test]
async fn anthropic_routes_share_the_request_body_limit() {
let web = FakeServer::start(|_| async {
diff --git a/ai-gateway/src/providers/chat_completions.rs b/ai-gateway/src/providers/chat_completions.rs
new file mode 100644
index 0000000000..f1ad94e254
--- /dev/null
+++ b/ai-gateway/src/providers/chat_completions.rs
@@ -0,0 +1,147 @@
+//! Chat Completions only reports streamed usage when the caller opts in, so the
+//! gateway opts in on the caller's behalf. The resulting usage chunk is relayed.
+use std::fmt;
+
+use axum::{body::Bytes, http::HeaderMap};
+use serde::{
+ Deserialize, Deserializer,
+ de::{MapAccess, Visitor},
+};
+use serde_json::{Value, value::RawValue};
+
+use crate::transport::identity_encoding;
+
+/// The rewritten body for a streamed request that did not ask for usage, or
+/// `None` when the original bytes must be forwarded unchanged. Every other
+/// top-level member keeps its original bytes and position.
+pub(super) fn request_stream_usage(headers: &HeaderMap, body: &[u8]) -> Option<Bytes> {
+ if !identity_encoding(headers) {
+ return None;
+ }
+ let Members(mut members) = serde_json::from_slice(body).ok()?;
+ // Duplicate members are resolved differently by different parsers.
+ if count(&members, "stream") != 1 || count(&members, "stream_options") > 1 {
+ return None;
+ }
+ if member(&members, "stream")? != "true" {
+ return None;
+ }
+ let options = match member(&members, "stream_options") {
+ None => serde_json::Map::new(),
+ Some(raw) => match serde_json::from_str::<Value>(raw).ok()? {
+ Value::Object(options) => options,
+ Value::Null => serde_json::Map::new(),
+ _ => return None,
+ },
+ };
+ if options.get("include_usage") == Some(&Value::Bool(true)) {
+ return None;
+ }
+ let mut options = options;
+ options.insert("include_usage".to_owned(), Value::Bool(true));
+ let options = RawValue::from_string(Value::Object(options).to_string()).ok()?;
+ match members.iter_mut().find(|(key, _)| key == "stream_options") {
+ Some((_, raw)) => *raw = options,
+ None => members.push(("stream_options".to_owned(), options)),
+ }
+ let mut rewritten = Vec::with_capacity(body.len() + 32);
+ rewritten.push(b'{');
+ for (index, (key, raw)) in members.iter().enumerate() {
+ if index > 0 {
+ rewritten.push(b',');
+ }
+ serde_json::to_writer(&mut rewritten, key).ok()?;
+ rewritten.push(b':');
+ rewritten.extend_from_slice(raw.get().as_bytes());
+ }
+ rewritten.push(b'}');
+ Some(Bytes::from(rewritten))
+}
+
+fn count(members: &[(String, Box<RawValue>)], name: &str) -> usize {
+ members.iter().filter(|(key, _)| key == name).count()
+}
+
+fn member<'a>(members: &'a [(String, Box<RawValue>)], name: &str) -> Option<&'a str> {
+ members
+ .iter()
+ .find(|(key, _)| key == name)
+ .map(|(_, raw)| raw.get())
+}
+
+/// A top-level JSON object in source order, with each value left unparsed.
+struct Members(Vec<(String, Box<RawValue>)>);
+
+impl<'de> Deserialize<'de> for Members {
+ fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+ struct MembersVisitor;
+ impl<'de> Visitor<'de> for MembersVisitor {
+ type Value = Members;
+ fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+ formatter.write_str("a JSON object")
+ }
+ fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Members, A::Error> {
+ let mut members = Vec::with_capacity(map.size_hint().unwrap_or(0));
+ while let Some(member) = map.next_entry()? {
+ members.push(member);
+ }
+ Ok(Members(members))
+ }
+ }
+ deserializer.deserialize_map(MembersVisitor)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use axum::http::{HeaderValue, header};
+
+ fn rewrite(body: &str) -> Option<String> {
+ request_stream_usage(&HeaderMap::new(), body.as_bytes())
+ .map(|bytes| String::from_utf8(bytes.to_vec()).unwrap())
+ }
+
+ #[test]
+ fn streamed_requests_gain_include_usage_without_touching_other_members() {
+ assert_eq!(
+ rewrite(r#"{"model":"gpt-4.1", "messages":[{"role":"user","content":"1.0e2 \u00e9"}],"stream":true}"#)
+ .as_deref(),
+ Some(
+ r#"{"model":"gpt-4.1","messages":[{"role":"user","content":"1.0e2 \u00e9"}],"stream":true,"stream_options":{"include_usage":true}}"#
+ )
+ );
+ assert_eq!(
+ rewrite(r#"{"stream_options":{"include_obfuscation":false,"include_usage":false},"stream":true,"model":"m"}"#)
+ .as_deref(),
+ Some(
+ r#"{"stream_options":{"include_obfuscation":false,"include_usage":true},"stream":true,"model":"m"}"#
+ )
+ );
+ assert_eq!(
+ rewrite(r#"{"stream":true,"stream_options":null}"#).as_deref(),
+ Some(r#"{"stream":true,"stream_options":{"include_usage":true}}"#)
+ );
+ }
+
+ #[test]
+ fn other_bodies_are_forwarded_unchanged() {
+ for body in [
+ r#"{"model":"m","messages":[]}"#,
+ r#"{"stream":false}"#,
+ r#"{"stream":"true"}"#,
+ r#"{"stream":true,"stream_options":{"include_usage":true}}"#,
+ r#"{"stream":true,"stream_options":"invalid"}"#,
+ r#"{"stream":true,"stream":true}"#,
+ r#"{"stream":true,"stream_options":{},"stream_options":{}}"#,
+ r#"[{"stream":true}]"#,
+ r#"{"stream":true"#,
+ "",
+ ] {
+ assert_eq!(rewrite(body), None, "{body}");
+ }
+ let mut encoded = HeaderMap::new();
+ encoded.insert(header::CONTENT_ENCODING, HeaderValue::from_static("gzip"));
+ assert!(request_stream_usage(&encoded, br#"{"stream":true}"#).is_none());
+ }
+}
diff --git a/ai-gateway/src/providers/mod.rs b/ai-gateway/src/providers/mod.rs
index ace07407e4..5e5871627b 100644
--- a/ai-gateway/src/providers/mod.rs
+++ b/ai-gateway/src/providers/mod.rs
@@ -1,3 +1,5 @@
+mod chat_completions;
+
use std::{sync::Arc, time::Duration};
use axum::{
@@ -24,6 +26,7 @@ use crate::{
pub(crate) enum Route {
OpenAiResponses,
OpenAiResponsesCompact,
+ OpenAiChatCompletions,
OpenAiModels,
AnthropicMessages,
AnthropicCountTokens,
@@ -33,9 +36,10 @@ pub(crate) enum Route {
impl Route {
pub(crate) fn provider(self) -> Provider {
match self {
- Self::OpenAiResponses | Self::OpenAiResponsesCompact | Self::OpenAiModels => {
- Provider::OpenAi
- }
+ Self::OpenAiResponses
+ | Self::OpenAiResponsesCompact
+ | Self::OpenAiChatCompletions
+ | Self::OpenAiModels => Provider::OpenAi,
Self::AnthropicMessages | Self::AnthropicCountTokens | Self::AnthropicModels => {
Provider::Anthropic
}
@@ -43,9 +47,12 @@ impl Route {
}
pub(crate) fn api_format(self) -> ApiFormat {
- match self.provider() {
- Provider::OpenAi => ApiFormat::OpenAiResponses,
- Provider::Anthropic => ApiFormat::AnthropicMessages,
+ match self {
+ Self::OpenAiChatCompletions => ApiFormat::OpenAiChatCompletions,
+ _ => match self.provider() {
+ Provider::OpenAi => ApiFormat::OpenAiResponses,
+ Provider::Anthropic => ApiFormat::AnthropicMessages,
+ },
}
}
@@ -60,6 +67,7 @@ impl Route {
match self {
Self::OpenAiResponses => "/responses",
Self::OpenAiResponsesCompact => "/responses/compact",
+ Self::OpenAiChatCompletions => "/chat/completions",
Self::OpenAiModels | Self::AnthropicModels => "/models",
Self::AnthropicMessages => "/messages",
Self::AnthropicCountTokens => "/messages/count_tokens",
@@ -69,7 +77,10 @@ impl Route {
pub(crate) fn captures_generation(self) -> bool {
matches!(
self,
- Self::OpenAiResponses | Self::OpenAiResponsesCompact | Self::AnthropicMessages
+ Self::OpenAiResponses
+ | Self::OpenAiResponsesCompact
+ | Self::OpenAiChatCompletions
+ | Self::AnthropicMessages
)
}
@@ -215,6 +226,13 @@ impl ProviderTransport {
} else {
ExecutionCapture::unobserved()
};
+ // The capture keeps the caller's own request; only the upstream copy asks for usage.
+ let body = match route {
+ Route::OpenAiChatCompletions => {
+ chat_completions::request_stream_usage(headers, &body).unwrap_or(body)
+ }
+ _ => body,
+ };
let mut upstream = self
.client
.request(route.method(), self.request_url(route, query))
diff --git a/ai-gateway/src/providers/tests.rs b/ai-gateway/src/providers/tests.rs
index 2f7382672a..85038269a0 100644
--- a/ai-gateway/src/providers/tests.rs
+++ b/ai-gateway/src/providers/tests.rs
@@ -961,3 +961,223 @@ fn openai_and_anthropic_admissions_draw_from_one_execution_budget() {
drop(held);
assert!(relay.try_admit().is_ok());
}
+
+const CHAT_CONTENT_CHUNK: &str = "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"gpt-4.1-mini-2025-04-14\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hello\"},\"finish_reason\":\"stop\"}],\"usage\":null}\n\n";
+const CHAT_USAGE_CHUNK: &str = "data: {\"id\":\"chatcmpl-1\",\"object\":\"chat.completion.chunk\",\"model\":\"gpt-4.1-mini-2025-04-14\",\"choices\":[],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":1,\"total_tokens\":10,\"prompt_tokens_details\":{\"cached_tokens\":4},\"completion_tokens_details\":{\"reasoning_tokens\":0}}}\n\n";
+const CHAT_DONE: &str = "data: [DONE]\n\n";
+
+/// Generation context headers in the shape SDK users send them.
+const CHAT_CONTEXT_HEADERS: [(&str, &str); 8] = [
+ (
+ "traceparent",
+ "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
+ ),
+ ("langfuse-user-id", "chat-user"),
+ ("langfuse-session-id", "chat-session"),
+ ("langfuse-trace-name", "chat-trace"),
+ ("langfuse-environment", "staging"),
+ ("langfuse-tags", "tag-a,tag-b"),
+ ("langfuse-metadata", "team:search"),
+ ("content-type", "application/json"),
+];
+
+#[tokio::test]
+async fn chat_completions_streams_request_usage_and_relay_the_usage_chunk() {
+ use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry, test_support::upload_json};
+
+ const UNREQUESTED: &str = r#"{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"Say hello"}],"stream":true}"#;
+ const REQUESTED: &str = r#"{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"Say hello"}],"stream":true,"stream_options":{"include_usage":true}}"#;
+ // Callers receive the usage chunk whether or not they asked for it.
+ let relayed = [CHAT_CONTENT_CHUNK, CHAT_USAGE_CHUNK, CHAT_DONE].concat();
+ for (request, forwarded) in [
+ (
+ UNREQUESTED,
+ r#"{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"Say hello"}],"stream":true,"stream_options":{"include_usage":true}}"#,
+ ),
+ (REQUESTED, REQUESTED),
+ ] {
+ let (sent, mut received) = tokio::sync::mpsc::channel(2);
+ let sink = FakeServer::start(move |request| {
+ let sent = sent.clone();
+ async move {
+ let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
+ sent.send(upload_json(&bytes)).await.unwrap();
+ Response::new(Body::from("{}"))
+ }
+ })
+ .await;
+ let telemetry =
+ Telemetry::for_test(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap());
+ let upstream = FakeServer::start(move |request| async move {
+ assert_eq!(request.uri(), "/v1/chat/completions");
+ assert_eq!(
+ request.headers()[header::AUTHORIZATION],
+ "Bearer provider-secret"
+ );
+ for (name, _) in CHAT_CONTEXT_HEADERS {
+ if name != "content-type" {
+ assert!(!request.headers().contains_key(name), "forwarded {name}");
+ }
+ }
+ assert_eq!(
+ to_bytes(request.into_body(), 4096).await.unwrap(),
+ forwarded
+ );
+ // Arrive split mid-event, as a provider stream may.
+ let native = [CHAT_CONTENT_CHUNK, CHAT_USAGE_CHUNK, CHAT_DONE].concat();
+ let (head, tail) = native.split_at(CHAT_CONTENT_CHUNK.len() + 20);
+ Response::builder()
+ .header("content-type", "text/event-stream; charset=utf-8")
+ .header("x-request-id", "req-chat")
+ .body(Body::from_stream(stream::iter([
+ Ok::<_, Infallible>(head.to_owned()),
+ Ok(tail.to_owned()),
+ ])))
+ .unwrap()
+ })
+ .await;
+ let relay = provider(&upstream, 1).with_telemetry(telemetry.clone());
+ let response = relay
+ .forward_route(
+ relay.try_admit().unwrap(),
+ resolved_request_context_for(
+ ApiFormat::OpenAiChatCompletions,
+ "provider-secret",
+ "full",
+ )
+ .await,
+ &headers(
+ &[
+ CHAT_CONTEXT_HEADERS.as_slice(),
+ &[("authorization", "Bearer gateway-secret")],
+ ]
+ .concat(),
+ ),
+ Bytes::from_static(request.as_bytes()),
+ Route::OpenAiChatCompletions,
+ None,
+ &mut crate::correlation::RequestCorrelation::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.headers()["x-request-id"], "req-chat");
+ assert_eq!(
+ to_bytes(response.into_body(), 65536).await.unwrap(),
+ relayed
+ );
+ let payload = tokio::time::timeout(Duration::from_secs(5), received.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ assert_chat_upload(&payload, request);
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(1))
+ .await;
+ }
+}
+
+#[tokio::test]
+async fn chat_completions_forward_non_streamed_bodies_unchanged() {
+ const JSON_REQUEST: &str = r#"{"model":"gpt-4.1-mini", "messages":[]}"#;
+ let upstream = FakeServer::start(|request| async move {
+ assert_eq!(
+ to_bytes(request.into_body(), 4096).await.unwrap(),
+ JSON_REQUEST
+ );
+ Response::builder()
+ .header("content-type", "application/json")
+ .body(Body::from(r#"{"id":"chatcmpl-1","choices":[]}"#))
+ .unwrap()
+ })
+ .await;
+ let relay = transport(&upstream);
+ let response = relay
+ .forward_route(
+ relay.try_admit().unwrap(),
+ resolved_request_context_for(
+ ApiFormat::OpenAiChatCompletions,
+ "provider-secret",
+ "usage",
+ )
+ .await,
+ &headers(&[("content-type", "application/json")]),
+ Bytes::from_static(JSON_REQUEST.as_bytes()),
+ Route::OpenAiChatCompletions,
+ None,
+ &mut crate::correlation::RequestCorrelation::default(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::OK);
+ assert_eq!(
+ to_bytes(response.into_body(), 4096).await.unwrap(),
+ r#"{"id":"chatcmpl-1","choices":[]}"#
+ );
+ assert_eq!(upstream.calls(), 1);
+}
+
+fn assert_chat_upload(payload: &serde_json::Value, request: &str) {
+ use serde_json::{Value, json};
+
+ let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];
+ assert_eq!(span["name"], "openai.chat-completions");
+ assert_eq!(span["traceId"], "4bf92f3577b34da6a3ce929d0e0e4736");
+ assert_eq!(span["parentSpanId"], "00f067aa0ba902b7");
+ let attribute = |key: &str| {
+ let attribute = span["attributes"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|attribute| attribute["key"] == key)
+ .unwrap_or_else(|| panic!("missing {key}"));
+ attribute["value"]["stringValue"].clone()
+ };
+ let parsed =
+ |key: &str| -> Value { serde_json::from_str(attribute(key).as_str().unwrap()).unwrap() };
+ for (key, value) in [
+ ("user.id", "chat-user"),
+ ("session.id", "chat-session"),
+ ("langfuse.trace.name", "chat-trace"),
+ ("langfuse.environment", "staging"),
+ ("langfuse.observation.level", "DEFAULT"),
+ ("langfuse.observation.model.name", "gpt-4.1-mini-2025-04-14"),
+ ] {
+ assert_eq!(attribute(key), value, "{key}");
+ }
+ assert_eq!(parsed("langfuse.trace.tags"), json!(["tag-a", "tag-b"]));
+ assert_eq!(
+ parsed("langfuse.observation.usage_details"),
+ json!({"prompt_tokens":9,"completion_tokens":1,"total_tokens":10,"prompt_tokens_details":{"cached_tokens":4},"completion_tokens_details":{"reasoning_tokens":0}})
+ );
+ // The recorded input is the caller's request, not the upstream rewrite.
+ assert_eq!(
+ parsed("langfuse.observation.input"),
+ json!({"messages":[{"role":"user","content":"Say hello"}]})
+ );
+ assert_eq!(
+ parsed("langfuse.observation.model.parameters").get("stream_options"),
+ serde_json::from_str::<Value>(request)
+ .unwrap()
+ .get("stream_options")
+ );
+ assert_eq!(
+ parsed("langfuse.observation.output"),
+ json!({"choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"Hello"}}]})
+ );
+ attribute("langfuse.observation.completion_start_time");
+ let metadata = parsed("langfuse.observation.metadata");
+ assert_eq!(metadata["team"], "search");
+ assert_eq!(
+ metadata["langfuse.gateway.request.api_format"],
+ "openai.chat-completions"
+ );
+ assert_eq!(metadata["langfuse.gateway.response.id"], "chatcmpl-1");
+ assert_eq!(metadata["langfuse.gateway.upstream.request.id"], "req-chat");
+ for secret in [
+ "provider-secret",
+ "gateway-secret",
+ "private-ingestion-token",
+ ] {
+ assert!(!payload.to_string().contains(secret));
+ }
+}
diff --git a/ai-gateway/src/resolution/contracts.rs b/ai-gateway/src/resolution/contracts.rs
index 581e1166a8..c296f2f00c 100644
--- a/ai-gateway/src/resolution/contracts.rs
+++ b/ai-gateway/src/resolution/contracts.rs
@@ -10,6 +10,8 @@ use std::{collections::BTreeMap, fmt};
pub enum ApiFormat {
#[serde(rename = "openai.responses")]
OpenAiResponses,
+ #[serde(rename = "openai.chat-completions")]
+ OpenAiChatCompletions,
#[serde(rename = "anthropic.messages")]
AnthropicMessages,
}
@@ -32,8 +34,10 @@ impl Provider {
fn supports(self, api_format: ApiFormat) -> bool {
matches!(
(self, api_format),
- (Self::OpenAi, ApiFormat::OpenAiResponses)
- | (Self::Anthropic, ApiFormat::AnthropicMessages)
+ (
+ Self::OpenAi,
+ ApiFormat::OpenAiResponses | ApiFormat::OpenAiChatCompletions
+ ) | (Self::Anthropic, ApiFormat::AnthropicMessages)
)
}
diff --git a/ai-gateway/src/resolution/tests.rs b/ai-gateway/src/resolution/tests.rs
index 45dd611884..59aa61e038 100644
--- a/ai-gateway/src/resolution/tests.rs
+++ b/ai-gateway/src/resolution/tests.rs
@@ -427,6 +427,54 @@ async fn resolves_anthropic_messages_with_the_native_credential_scheme() {
assert_eq!(web.calls(), 1);
}
+#[tokio::test]
+async fn chat_completions_resolve_only_an_openai_connection_for_that_format() {
+ let mut chat = success("project-1", "provider-token");
+ chat["connection"]["api_format"] = json!("openai.chat-completions");
+ let body = chat.clone();
+ let web = FakeWeb::start(move |request| {
+ let body = body.clone();
+ async move {
+ assert_eq!(
+ to_bytes(request.into_body(), 1024).await.unwrap(),
+ r#"{"apiFormat":"openai.chat-completions"}"#
+ );
+ response(body)
+ }
+ })
+ .await;
+ let context = web
+ .control_plane()
+ .resolve_at(
+ GATEWAY_KEY,
+ ApiFormat::OpenAiChatCompletions,
+ NOW_TIME,
+ REQUEST_ID,
+ )
+ .await
+ .unwrap();
+ assert_eq!(context.connection().provider(), Provider::OpenAi);
+ assert_eq!(
+ context.connection().api_format(),
+ ApiFormat::OpenAiChatCompletions
+ );
+ assert!(matches!(
+ context.connection().credential(),
+ ProviderCredential::Bearer("provider-token")
+ ));
+
+ // A Responses connection does not serve a Chat Completions request, or vice versa.
+ assert_invalid_context_for(
+ success("project-1", "provider-token"),
+ ApiFormat::OpenAiChatCompletions,
+ )
+ .await;
+ assert_invalid_context_for(chat.clone(), ApiFormat::OpenAiResponses).await;
+ let mut anthropic = anthropic_success("provider-key");
+ anthropic["connection"]["api_format"] = json!("openai.chat-completions");
+ assert_invalid_context_for(anthropic, ApiFormat::OpenAiChatCompletions).await;
+}
+
#[tokio::test]
async fn rejects_connections_whose_provider_format_origin_and_credential_disagree() {
// Web returned an Anthropic connection for an OpenAI Responses request, or the
diff --git a/ai-gateway/src/telemetry/mapping.rs b/ai-gateway/src/telemetry/mapping.rs
index 27515a20c3..2999d99f81 100644
--- a/ai-gateway/src/telemetry/mapping.rs
+++ b/ai-gateway/src/telemetry/mapping.rs
@@ -260,22 +260,31 @@ fn attribute(key: &str, value: impl Into<String>) -> Value {
fn usage_projection(api_format: &str, usage: &Value) -> Option<Value> {
match api_format {
- "openai.responses" => openai_usage(usage),
+ "openai.responses" => openai_usage(
+ usage,
+ ["input_tokens", "output_tokens", "total_tokens"],
+ ["input_tokens_details", "output_tokens_details"],
+ ),
+ "openai.chat-completions" => openai_usage(
+ usage,
+ ["prompt_tokens", "completion_tokens", "total_tokens"],
+ ["prompt_tokens_details", "completion_tokens_details"],
+ ),
"anthropic.messages" => usage.is_object().then(|| usage.clone()),
_ => None,
}
}
-/// The receiver's native `OpenAI` usage schema is strict at the top level, but
-/// accepts new numeric detail counters.
-fn openai_usage(usage: &Value) -> Option<Value> {
+/// The receiver's native `OpenAI` usage schemas are strict at the top level, but
+/// accept new numeric detail counters.
+fn openai_usage(usage: &Value, counters: [&str; 3], details: [&str; 2]) -> Option<Value> {
let usage = usage.as_object()?;
let mut projected = Map::new();
- for key in ["input_tokens", "output_tokens", "total_tokens"] {
+ for key in counters {
let value = usage.get(key).filter(|value| value.as_u64().is_some())?;
projected.insert(key.into(), value.clone());
}
- for key in ["input_tokens_details", "output_tokens_details"] {
+ for key in details {
match usage.get(key) {
Some(Value::Null) => {
projected.insert(key.into(), Value::Null);
@@ -609,6 +618,39 @@ mod tests {
);
}
+ #[test]
+ fn chat_completions_usage_projects_into_the_receivers_completion_schema() {
+ let native = json!({
+ "prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30,
+ "prompt_tokens_details": {"cached_tokens": 4, "audio_tokens": 0},
+ "completion_tokens_details": {"reasoning_tokens": 6, "accepted_prediction_tokens": null, "future_object": {}},
+ "future_usage": {"cost": 123},
+ });
+ assert_eq!(
+ projected_usage("openai.chat-completions", native),
+ Some(json!({
+ "prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30,
+ "prompt_tokens_details": {"cached_tokens": 4, "audio_tokens": 0},
+ "completion_tokens_details": {"reasoning_tokens": 6, "accepted_prediction_tokens": null},
+ }))
+ );
+ // Each OpenAI contract projects only its own counter names.
+ assert_eq!(
+ projected_usage(
+ "openai.chat-completions",
+ json!({"input_tokens": 2, "output_tokens": 1, "total_tokens": 3})
+ ),
+ None
+ );
+ assert_eq!(
+ projected_usage(
+ "openai.responses",
+ json!({"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3})
+ ),
+ None
+ );
+ }
+
#[test]
fn provider_request_metadata_is_namespaced_and_keeps_native_types() {
let mut facts = facts();
diff --git a/ai-gateway/src/test_support.rs b/ai-gateway/src/test_support.rs
index a078ce8837..23606f7f9b 100644
--- a/ai-gateway/src/test_support.rs
+++ b/ai-gateway/src/test_support.rs
@@ -103,6 +103,11 @@ pub(crate) fn resolution_response_for(
"base_url": "https://api.openai.com/v1",
"auth": {"type": "Bearer", "token": provider_secret}
}),
+ ApiFormat::OpenAiChatCompletions => json!({
+ "id": "connection-1", "provider": "openai", "api_format": "openai.chat-completions",
+ "base_url": "https://api.openai.com/v1",
+ "auth": {"type": "Bearer", "token": provider_secret}
+ }),
ApiFormat::AnthropicMessages => json!({
"id": "connection-1", "provider": "anthropic", "api_format": "anthropic.messages",
"base_url": "https://api.anthropic.com/v1",
diff --git a/ai-gateway/src/transport/mod.rs b/ai-gateway/src/transport/mod.rs
index c543695363..54de712f93 100644
--- a/ai-gateway/src/transport/mod.rs
+++ b/ai-gateway/src/transport/mod.rs
@@ -82,7 +82,9 @@ const COMMON_RESPONSE_HEADERS: &[&str] = &[
pub(crate) fn request_headers(source: &HeaderMap, api_format: ApiFormat) -> HeaderMap {
match api_format {
- ApiFormat::OpenAiResponses => selected_headers(source, COMMON_REQUEST_HEADERS, &[]),
+ ApiFormat::OpenAiResponses | ApiFormat::OpenAiChatCompletions => {
+ selected_headers(source, COMMON_REQUEST_HEADERS, &[])
+ }
ApiFormat::AnthropicMessages => {
selected_headers(source, COMMON_REQUEST_HEADERS, &["anthropic-"])
}
@@ -91,7 +93,7 @@ pub(crate) fn request_headers(source: &HeaderMap, api_format: ApiFormat) -> Head
pub(crate) fn response_headers(source: &HeaderMap, api_format: ApiFormat) -> HeaderMap {
match api_format {
- ApiFormat::OpenAiResponses => selected_headers(
+ ApiFormat::OpenAiResponses | ApiFormat::OpenAiChatCompletions => selected_headers(
source,
&[
COMMON_RESPONSE_HEADERS,
@@ -125,7 +127,7 @@ pub(crate) fn response_headers(source: &HeaderMap, api_format: ApiFormat) -> Hea
/// The provider's own request ID, bounded like other captured provider facts.
pub(crate) fn provider_request_id(headers: &HeaderMap, api_format: ApiFormat) -> Option<&str> {
let name = match api_format {
- ApiFormat::OpenAiResponses => "x-request-id",
+ ApiFormat::OpenAiResponses | ApiFormat::OpenAiChatCompletions => "x-request-id",
ApiFormat::AnthropicMessages => "request-id",
};
headers
@@ -135,6 +137,17 @@ pub(crate) fn provider_request_id(headers: &HeaderMap, api_format: ApiFormat) ->
.filter(|value| value.len() <= 512)
}
+pub(crate) fn identity_encoding(headers: &HeaderMap) -> bool {
+ headers
+ .get_all(header::CONTENT_ENCODING)
+ .iter()
+ .all(|value| {
+ value
+ .to_str()
+ .is_ok_and(|value| value.eq_ignore_ascii_case("identity"))
+ })
+}
+
pub(crate) fn relay<T: Send + 'static>(
upstream: reqwest::Response,
deadline: Instant,