Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): capture full-mode inputs up to 5 MiB and mark omitted inputs (#17960)

Hassieb Pakzadae9e08aaf3a5
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index 140c418b17..5ad29a91e4 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -393,9 +393,10 @@ with the available HTTP status in its status message. Full mode also includes a
bounded provider error code/message; usage mode omits these details because provider
errors may echo request content.
-Provisional upload limits are 32 concurrent uploads, 1024 queued records, 4 MiB
-serialized span and credentials per record, 64 MiB total retained span/credential
-bytes by default (`LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES`), 8 MiB of OTLP JSON per
+Provisional upload limits are 32 concurrent uploads, 1024 queued records, 16 MiB
+serialized span and credentials per record (a 5 MiB input plus 1 MiB output, each
+possibly doubled by JSON string escaping), 64 MiB total retained span/credential
+bytes by default (`LANGFUSE_AI_GATEWAY_TELEMETRY_BUFFER_BYTES`), 20 MiB of OTLP JSON per
payload before gzip compression, and
64 KiB ingestion responses. Serialization and mapping have additional bounded memory
overhead; these byte budgets are not an RSS limit. Uploads have a two-second connect
@@ -620,10 +621,18 @@ hands an owned `InferenceFacts` record to telemetry for a safe debug summary and
batched upload. Capture and upload run independently of log level; debug emission
alone is gated.
-Capture is limited to 1 MiB request inspection, 1 MiB JSON/SSE event inspection,
+Capture is limited to 5 MiB request inspection, 1 MiB JSON/SSE event inspection,
1 MiB retained output and 256 output items per execution. The active-request limit
bounds the number of captures. Trusted key metadata is bounded by the resolver's
-256 KiB response limit. Oversized input is omitted; oversized output items
+256 KiB response limit. In full mode, an input that is not recorded (over 5 MiB,
+content-encoded or not a JSON object) is explained in generation metadata:
+`langfuse.gateway.request.input_omitted` (`size_limit`, `content_encoding` or
+`invalid_json`), `langfuse.gateway.request.body_bytes`, and, for `size_limit`,
+`langfuse.gateway.request.input_limit_bytes`. If a generation with its input would
+exceed the telemetry record limit (capped at the retained buffer size) or the buffer
+is full, the input is dropped from that record and the generation is still uploaded,
+with `input_omitted` set to `record_limit` or `telemetry_buffer` and
+`langfuse.gateway.request.input_bytes`. Oversized output items
are skipped and completeness is false. Malformed, truncated or compressed bodies
do not interrupt the relay. Capture buffers are independent of forwarding, so
these limits never cap the actual provider response. Media is not fetched/uploaded.
diff --git a/ai-gateway/src/capture/anthropic_messages.rs b/ai-gateway/src/capture/anthropic_messages.rs
index 00d66560d6..7a7f44f450 100644
--- a/ai-gateway/src/capture/anthropic_messages.rs
+++ b/ai-gateway/src/capture/anthropic_messages.rs
@@ -1,6 +1,6 @@
use super::{
- MAX_CAPTURE_BYTES, MAX_FACT_STRING, MAX_ITEMS, bounded_string, facts::ProviderFacts,
- identity_encoding, response::ResponseBody,
+ MAX_FACT_STRING, MAX_ITEMS, MAX_OUTPUT_CAPTURE_BYTES, bounded_string, facts::ProviderFacts,
+ parse_request, response::ResponseBody,
};
use crate::resolution::IngestionMode;
use axum::http::HeaderMap;
@@ -192,7 +192,7 @@ impl AssistantMessage {
fn reserve(used: &mut usize, bytes: usize) -> Result<(), Gap> {
match used.checked_add(bytes) {
- Some(total) if total <= MAX_CAPTURE_BYTES => {
+ Some(total) if total <= MAX_OUTPUT_CAPTURE_BYTES => {
*used = total;
Ok(())
}
@@ -216,12 +216,15 @@ impl AnthropicMessagesCapture {
request_complete: false,
response_valid: true,
};
- if identity_encoding(headers)
- && body.len() <= MAX_CAPTURE_BYTES
- && let Ok(Value::Object(request)) = serde_json::from_slice(body)
- {
- capture.capture_request(request);
- capture.request_complete = true;
+ match parse_request(headers, body) {
+ Ok(request) => {
+ capture.capture_request(request);
+ capture.request_complete = true;
+ }
+ Err(omission) if mode == IngestionMode::Full => {
+ capture.facts.input_omission = Some(omission);
+ }
+ Err(_) => {}
}
capture
}
diff --git a/ai-gateway/src/capture/anthropic_messages/tests.rs b/ai-gateway/src/capture/anthropic_messages/tests.rs
index aed48421a3..70ef8c5abd 100644
--- a/ai-gateway/src/capture/anthropic_messages/tests.rs
+++ b/ai-gateway/src/capture/anthropic_messages/tests.rs
@@ -1,6 +1,9 @@
use super::*;
use crate::{
- capture::{ExecutionCapture, ProtocolCapture, ProviderFacts},
+ capture::{
+ ExecutionCapture, InputOmission, InputOmissionReason, MAX_INPUT_CAPTURE_BYTES,
+ ProtocolCapture, ProviderFacts,
+ },
providers::{ProviderLimits, ProviderTransport, Route},
resolution::ApiFormat,
test_support::{FakeServer, resolved_request_context_for},
@@ -602,3 +605,61 @@ async fn streamed_messages_upload_one_generation_with_native_usage() {
assert!(!serialized.contains(canary), "usage mode uploaded {canary}");
}
}
+
+#[tokio::test]
+async fn full_mode_marks_requests_it_cannot_record_as_input() {
+ let at_limit = format!(
+ r#"{{"model":"claude-sonnet-4-5","messages":[{{"role":"user","content":"{}"}}]}}"#,
+ "x".repeat(MAX_INPUT_CAPTURE_BYTES - 128)
+ );
+ let over_limit = format!(
+ "{at_limit}{}",
+ " ".repeat(MAX_INPUT_CAPTURE_BYTES - at_limit.len() + 1)
+ );
+ assert!(at_limit.len() <= MAX_INPUT_CAPTURE_BYTES);
+ let mut gzip = HeaderMap::new();
+ gzip.insert(header::CONTENT_ENCODING, HeaderValue::from_static("gzip"));
+ for (mode, headers, body, omission) in [
+ ("full", HeaderMap::new(), at_limit.as_str(), None),
+ (
+ "full",
+ HeaderMap::new(),
+ over_limit.as_str(),
+ Some(InputOmissionReason::SizeLimit),
+ ),
+ (
+ "full",
+ gzip.clone(),
+ at_limit.as_str(),
+ Some(InputOmissionReason::ContentEncoding),
+ ),
+ (
+ "full",
+ HeaderMap::new(),
+ "not json",
+ Some(InputOmissionReason::InvalidJson),
+ ),
+ ("usage", HeaderMap::new(), over_limit.as_str(), None),
+ ] {
+ let context =
+ resolved_request_context_for(ApiFormat::AnthropicMessages, "provider-secret", mode)
+ .await;
+ let observer = ExecutionCapture::for_request(
+ ApiFormat::AnthropicMessages,
+ &context,
+ &headers,
+ body.as_bytes(),
+ );
+ let facts = &captured(&observer).facts;
+ assert_eq!(
+ facts.input_omission,
+ omission.map(|reason| InputOmission {
+ reason,
+ body_bytes: body.len(),
+ }),
+ "{mode} {} bytes",
+ body.len()
+ );
+ assert_eq!(facts.input.is_some(), mode == "full" && omission.is_none());
+ }
+}
diff --git a/ai-gateway/src/capture/facts.rs b/ai-gateway/src/capture/facts.rs
index b1715c4820..2e57e62ecd 100644
--- a/ai-gateway/src/capture/facts.rs
+++ b/ai-gateway/src/capture/facts.rs
@@ -22,6 +22,22 @@ impl RelayOutcome {
}
}
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum InputOmissionReason {
+ SizeLimit,
+ ContentEncoding,
+ InvalidJson,
+ RecordLimit,
+ TelemetryBuffer,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
+pub(crate) struct InputOmission {
+ pub reason: InputOmissionReason,
+ pub body_bytes: usize,
+}
+
#[derive(Serialize)]
pub(crate) struct InferenceFacts {
pub api_format: &'static str,
@@ -46,6 +62,7 @@ pub(crate) struct ProviderFacts {
pub error_message: Option<String>,
pub usage_details: Option<Value>,
pub input: Option<Value>,
+ pub input_omission: Option<InputOmission>,
pub output: Option<Value>,
pub provider_response_id: Option<String>,
pub provider_request_id: Option<String>,
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rs
index 68c141d5be..854affaefb 100644
--- a/ai-gateway/src/capture/mod.rs
+++ b/ai-gateway/src/capture/mod.rs
@@ -16,10 +16,11 @@ use crate::{
};
use anthropic_messages::AnthropicMessagesCapture;
pub(crate) use facts::ProviderFacts;
-pub(crate) use facts::{InferenceFacts, RelayOutcome};
+pub(crate) use facts::{InferenceFacts, InputOmission, InputOmissionReason, RelayOutcome};
use openai_responses::OpenAiResponsesCapture;
-const MAX_CAPTURE_BYTES: usize = 1024 * 1024;
+pub(crate) const MAX_INPUT_CAPTURE_BYTES: usize = 5 * 1024 * 1024;
+pub(crate) const MAX_OUTPUT_CAPTURE_BYTES: usize = 1024 * 1024;
const MAX_ITEMS: usize = 256;
const MAX_FACT_STRING: usize = 512;
@@ -237,6 +238,22 @@ impl Drop for ExecutionCapture {
}
}
+fn parse_request(headers: &HeaderMap, body: &[u8]) -> Result<Map<String, Value>, InputOmission> {
+ let reason = if !identity_encoding(headers) {
+ InputOmissionReason::ContentEncoding
+ } else if body.len() > MAX_INPUT_CAPTURE_BYTES {
+ InputOmissionReason::SizeLimit
+ } else if let Ok(Value::Object(request)) = serde_json::from_slice(body) {
+ return Ok(request);
+ } else {
+ InputOmissionReason::InvalidJson
+ };
+ Err(InputOmission {
+ reason,
+ body_bytes: body.len(),
+ })
+}
+
fn bounded_string(value: &str) -> Option<String> {
(value.len() <= MAX_FACT_STRING).then(|| value.to_owned())
}
diff --git a/ai-gateway/src/capture/openai_responses.rs b/ai-gateway/src/capture/openai_responses.rs
index 968e36d289..5270cf489e 100644
--- a/ai-gateway/src/capture/openai_responses.rs
+++ b/ai-gateway/src/capture/openai_responses.rs
@@ -1,7 +1,7 @@
//! `OpenAI` Responses request, JSON response and completed SSE item capture.
use super::{
- MAX_CAPTURE_BYTES, MAX_FACT_STRING, MAX_ITEMS, bounded_string, facts::ProviderFacts,
- identity_encoding, response::ResponseBody,
+ MAX_FACT_STRING, MAX_ITEMS, MAX_OUTPUT_CAPTURE_BYTES, bounded_string, facts::ProviderFacts,
+ parse_request, response::ResponseBody,
};
use crate::{resolution::IngestionMode, telemetry};
use axum::http::HeaderMap;
@@ -35,12 +35,15 @@ impl OpenAiResponsesCapture {
response_valid: true,
client_metadata: None,
};
- if identity_encoding(headers)
- && body.len() <= MAX_CAPTURE_BYTES
- && let Ok(Value::Object(request)) = serde_json::from_slice(body)
- {
- capture.capture_request(request);
- capture.request_complete = true;
+ match parse_request(headers, body) {
+ Ok(request) => {
+ capture.capture_request(request);
+ capture.request_complete = true;
+ }
+ Err(omission) if mode == IngestionMode::Full => {
+ capture.facts.input_omission = Some(omission);
+ }
+ Err(_) => {}
}
capture
}
@@ -252,7 +255,7 @@ impl OpenAiResponsesCapture {
.items
.get(&index)
.map_or(0, |item| item.to_string().len());
- if self.output_bytes - previous + size > MAX_CAPTURE_BYTES
+ if self.output_bytes - previous + size > MAX_OUTPUT_CAPTURE_BYTES
|| (!self.items.contains_key(&index) && self.items.len() >= MAX_ITEMS)
{
self.response_valid = false;
diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rs
index e854997c38..19fe30c9a9 100644
--- a/ai-gateway/src/capture/openai_responses/tests.rs
+++ b/ai-gateway/src/capture/openai_responses/tests.rs
@@ -1,5 +1,8 @@
use super::*;
-use crate::capture::{ExecutionCapture, ProtocolCapture, RelayOutcome, sse::SseDecoder};
+use crate::capture::{
+ ExecutionCapture, InputOmission, InputOmissionReason, MAX_INPUT_CAPTURE_BYTES, ProtocolCapture,
+ RelayOutcome, sse::SseDecoder,
+};
use crate::resolution::ApiFormat;
use crate::{
providers::{ProviderLimits, ProviderTransport},
@@ -357,7 +360,8 @@ async fn oversized_and_malformed_events_do_not_prevent_later_facts() {
let mut observer = observer("full").await;
record_response(&mut observer, "text/event-stream");
observer.push_bytes(b"data: malformed\n\n");
- observer.push_bytes(format!("data: {}\n\n", "x".repeat(MAX_CAPTURE_BYTES + 1)).as_bytes());
+ observer
+ .push_bytes(format!("data: {}\n\n", "x".repeat(MAX_OUTPUT_CAPTURE_BYTES + 1)).as_bytes());
observer.push_bytes(terminal("completed", 0).as_bytes());
observer.end_body();
let capture = captured(&observer);
@@ -862,7 +866,7 @@ async fn codex_body_metadata_reaches_the_generation_without_agent_headers() {
#[tokio::test]
async fn capture_regression_large_request_does_not_invalidate_output() {
let context = resolved_request_context_with_mode("provider-secret", "full").await;
- let request = json!({"input": "x".repeat(MAX_CAPTURE_BYTES)}).to_string();
+ let request = json!({"input": "x".repeat(MAX_INPUT_CAPTURE_BYTES)}).to_string();
for streaming in [false, true] {
let mut observer = ExecutionCapture::for_request(
ApiFormat::OpenAiResponses,
@@ -887,6 +891,14 @@ async fn capture_regression_large_request_does_not_invalidate_output() {
observer.end_body();
let facts = &captured(&observer).facts;
assert!(!facts.input_complete);
+ assert!(facts.input.is_none());
+ assert_eq!(
+ facts.input_omission,
+ Some(InputOmission {
+ reason: InputOmissionReason::SizeLimit,
+ body_bytes: request.len(),
+ })
+ );
assert!(
facts.output_complete,
"response capture must be independent of request size"
@@ -894,6 +906,29 @@ async fn capture_regression_large_request_does_not_invalidate_output() {
}
}
+#[tokio::test]
+async fn request_at_input_limit_is_captured_in_full() {
+ let context = resolved_request_context_with_mode("provider-secret", "full").await;
+ let request = json!({"input": "x".repeat(MAX_INPUT_CAPTURE_BYTES - 12)}).to_string();
+ assert_eq!(request.len(), MAX_INPUT_CAPTURE_BYTES);
+ let observer = ExecutionCapture::for_request(
+ ApiFormat::OpenAiResponses,
+ &context,
+ &HeaderMap::new(),
+ request.as_bytes(),
+ );
+ let facts = &captured(&observer).facts;
+ assert!(facts.input_complete);
+ assert!(facts.input_omission.is_none());
+ assert_eq!(
+ facts.input.as_ref().unwrap()["input"]
+ .as_str()
+ .unwrap()
+ .len(),
+ MAX_INPUT_CAPTURE_BYTES - 12
+ );
+}
+
#[tokio::test]
async fn capture_regression_preserves_native_request_and_usage() {
let usage = json!({"input_tokens":5,"output_tokens":2,"input_tokens_details":{"cached_tokens":3,"future_tokens":1},"future_usage_field":{"region":"eu"}});
diff --git a/ai-gateway/src/capture/response.rs b/ai-gateway/src/capture/response.rs
index bfcbc768ba..43d04971a0 100644
--- a/ai-gateway/src/capture/response.rs
+++ b/ai-gateway/src/capture/response.rs
@@ -1,4 +1,4 @@
-use super::{MAX_CAPTURE_BYTES, identity_encoding, sse::SseDecoder};
+use super::{MAX_OUTPUT_CAPTURE_BYTES, identity_encoding, sse::SseDecoder};
use axum::http::{HeaderMap, header};
pub(super) enum ResponseBody {
@@ -32,10 +32,12 @@ impl ResponseBody {
pub(super) fn push(&mut self, bytes: &[u8], on_event: impl FnMut(&[u8])) -> bool {
match self {
Self::Sse(sse) => {
- sse.push(bytes, MAX_CAPTURE_BYTES, on_event);
+ sse.push(bytes, MAX_OUTPUT_CAPTURE_BYTES, on_event);
true
}
- Self::Json(buffer) if buffer.len().saturating_add(bytes.len()) <= MAX_CAPTURE_BYTES => {
+ Self::Json(buffer)
+ if buffer.len().saturating_add(bytes.len()) <= MAX_OUTPUT_CAPTURE_BYTES =>
+ {
buffer.extend_from_slice(bytes);
true
}
diff --git a/ai-gateway/src/telemetry/mapping.rs b/ai-gateway/src/telemetry/mapping.rs
index e2173b5834..89c184bf59 100644
--- a/ai-gateway/src/telemetry/mapping.rs
+++ b/ai-gateway/src/telemetry/mapping.rs
@@ -3,7 +3,7 @@ use chrono::{DateTime, SecondsFormat};
use serde_json::{Map, Value, json};
use super::context::GenerationContext;
-use crate::capture::{InferenceFacts, RelayOutcome};
+use crate::capture::{InferenceFacts, InputOmissionReason, MAX_INPUT_CAPTURE_BYTES, RelayOutcome};
pub(super) fn span(facts: InferenceFacts, context: &GenerationContext) -> Value {
let full = facts.metadata.get("ingestion_mode").and_then(Value::as_str) == Some("full");
@@ -91,6 +91,39 @@ pub(super) fn span(facts: InferenceFacts, context: &GenerationContext) -> Value
span
}
+pub(super) fn omit_input(span: &mut Value, reason: InputOmissionReason) -> bool {
+ let Some(attributes) = span["attributes"].as_array_mut() else {
+ return false;
+ };
+ let Some(index) = attributes
+ .iter()
+ .position(|attribute| attribute["key"] == "langfuse.observation.input")
+ else {
+ return false;
+ };
+ let input = attributes.remove(index);
+ let input_bytes = input["value"]["stringValue"].as_str().map_or(0, str::len);
+ if let Some(value) = attributes
+ .iter_mut()
+ .find(|attribute| attribute["key"] == "langfuse.observation.metadata")
+ .map(|attribute| &mut attribute["value"]["stringValue"])
+ && let Some(Value::Object(mut metadata)) = value
+ .as_str()
+ .and_then(|value| serde_json::from_str(value).ok())
+ {
+ metadata.insert(
+ "langfuse.gateway.request.input_omitted".into(),
+ json!(reason),
+ );
+ metadata.insert(
+ "langfuse.gateway.request.input_bytes".into(),
+ json!(input_bytes),
+ );
+ *value = Value::String(Value::Object(metadata).to_string());
+ }
+ true
+}
+
fn reserved_metadata(key: &str) -> bool {
[
"langfuse.gateway",
@@ -157,6 +190,22 @@ fn generation_metadata(facts: &InferenceFacts) -> Map<String, Value> {
for (key, value) in &facts.inference.request_metadata {
metadata.insert(format!("langfuse.gateway.request.{key}"), value.clone());
}
+ if let Some(omission) = facts.inference.input_omission {
+ metadata.insert(
+ "langfuse.gateway.request.input_omitted".into(),
+ json!(omission.reason),
+ );
+ metadata.insert(
+ "langfuse.gateway.request.body_bytes".into(),
+ json!(omission.body_bytes),
+ );
+ if omission.reason == InputOmissionReason::SizeLimit {
+ metadata.insert(
+ "langfuse.gateway.request.input_limit_bytes".into(),
+ json!(MAX_INPUT_CAPTURE_BYTES),
+ );
+ }
+ }
metadata
}
@@ -581,6 +630,47 @@ mod tests {
}
}
+ #[test]
+ fn omitted_input_is_explained_in_gateway_metadata() {
+ let mut facts = facts();
+ facts.inference.input = None;
+ facts.inference.input_omission = Some(crate::capture::InputOmission {
+ reason: InputOmissionReason::SizeLimit,
+ body_bytes: MAX_INPUT_CAPTURE_BYTES + 1,
+ });
+ let attrs = attributes(&span(facts, &context()));
+ assert!(!attrs.contains_key("langfuse.observation.input"));
+ let metadata = metadata(&attrs);
+ assert_eq!(
+ metadata["langfuse.gateway.request.input_omitted"],
+ "size_limit"
+ );
+ assert_eq!(
+ metadata["langfuse.gateway.request.body_bytes"],
+ MAX_INPUT_CAPTURE_BYTES + 1
+ );
+ assert_eq!(
+ metadata["langfuse.gateway.request.input_limit_bytes"],
+ MAX_INPUT_CAPTURE_BYTES
+ );
+
+ let mut facts = self::facts();
+ facts.inference.input_omission = Some(crate::capture::InputOmission {
+ reason: InputOmissionReason::ContentEncoding,
+ body_bytes: 10,
+ });
+ let metadata = self::metadata(&attributes(&span(facts, &context())));
+ assert_eq!(
+ metadata["langfuse.gateway.request.input_omitted"],
+ "content_encoding"
+ );
+ assert!(
+ metadata
+ .get("langfuse.gateway.request.input_limit_bytes")
+ .is_none()
+ );
+ }
+
#[test]
fn provider_errors_include_http_status_and_available_details() {
for (status, provider_status, error, expected) in [
diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rs
index 8f3e78435c..bc26140d65 100644
--- a/ai-gateway/src/telemetry/mod.rs
+++ b/ai-gateway/src/telemetry/mod.rs
@@ -26,7 +26,7 @@ use tokio::{
use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::{
- capture::InferenceFacts,
+ capture::{InferenceFacts, InputOmissionReason},
resolution::{ControlPlaneConfig, ResolutionError, ResolvedRequestContext},
};
use batch::{BatchPolicy, Batches, Pending};
@@ -37,7 +37,12 @@ use retry::RetryPolicy;
use worker::{Message, Uploads};
const MAX_UPLOADS: usize = 32;
-const MAX_RECORD_BYTES: usize = 4 * 1024 * 1024;
+const MAX_RECORD_BYTES: usize = 16 * 1024 * 1024;
+const _: () = assert!(
+ 2 * (crate::capture::MAX_INPUT_CAPTURE_BYTES + crate::capture::MAX_OUTPUT_CAPTURE_BYTES)
+ < MAX_RECORD_BYTES
+);
+const _: () = assert!(MAX_RECORD_BYTES < otlp::MAX_PAYLOAD_BYTES);
pub(crate) const DEFAULT_RETAINED_BYTES: usize = 64 * 1024 * 1024;
const MAX_QUEUED_RECORDS: usize = 1024;
@@ -114,6 +119,7 @@ fn crosses_report_threshold(before: u64, after: u64) -> bool {
struct Delivery {
queue: mpsc::Sender<Message>,
retained: Arc<Semaphore>,
+ record_limit: usize,
worker: Mutex<Option<JoinHandle<()>>>,
stats: Arc<Stats>,
}
@@ -169,6 +175,7 @@ impl Telemetry {
Self(Arc::new(Delivery {
queue,
retained: Arc::new(Semaphore::new(bytes)),
+ record_limit: MAX_RECORD_BYTES.min(bytes),
worker: Mutex::new(Some(worker)),
stats,
}))
@@ -180,23 +187,35 @@ impl Telemetry {
.span()
.span_context()
.clone();
- let span = mapping::span(facts, &context.generation);
+ let mut span = mapping::span(facts, &context.generation);
let credentials = context.grant.access_token.len() + context.grant.project_id.len();
- let mut size = SizeCounter {
- bytes: 0,
- limit: MAX_RECORD_BYTES.saturating_sub(credentials),
+ let limit = self.0.record_limit.saturating_sub(credentials);
+ let acquire = |bytes: usize| {
+ self.0.retained.clone().try_acquire_many_owned(
+ u32::try_from(bytes + credentials).expect("bounded record bytes"),
+ )
};
- if serde_json::to_writer(&mut size, &span).is_err() {
+ let mut size = serialized_size(&span, limit);
+ if size.is_none() && mapping::omit_input(&mut span, InputOmissionReason::RecordLimit) {
+ size = serialized_size(&span, limit);
+ }
+ let Some(mut bytes) = size else {
self.0.stats.record_dropped(1, "size");
return;
+ };
+ let mut retained = acquire(bytes).ok();
+ if retained.is_none()
+ && mapping::omit_input(&mut span, InputOmissionReason::TelemetryBuffer)
+ && let Some(reduced) = serialized_size(&span, limit)
+ {
+ bytes = reduced;
+ retained = acquire(bytes).ok();
}
- let Ok(retained) = self.0.retained.clone().try_acquire_many_owned(
- u32::try_from(size.bytes + credentials).expect("bounded record bytes"),
- ) else {
+ let Some(retained) = retained else {
self.0.stats.record_dropped(1, "bytes");
return;
};
- let item = Pending::new(span, size.bytes, link, retained);
+ let item = Pending::new(span, bytes, link, retained);
match self.0.queue.try_send(Message::Record(context.grant, item)) {
Ok(()) => {}
Err(TrySendError::Full(_)) => self.0.stats.record_dropped(1, "capacity"),
@@ -231,6 +250,12 @@ impl Telemetry {
}
}
+fn serialized_size(span: &Value, limit: usize) -> Option<usize> {
+ let mut size = SizeCounter { bytes: 0, limit };
+ serde_json::to_writer(&mut size, span).ok()?;
+ Some(size.bytes)
+}
+
/// Count serialized bytes without allocating a copy on the finalization path.
struct SizeCounter {
bytes: usize,
diff --git a/ai-gateway/src/telemetry/otlp.rs b/ai-gateway/src/telemetry/otlp.rs
index 71f7da88a8..0d7667e8e3 100644
--- a/ai-gateway/src/telemetry/otlp.rs
+++ b/ai-gateway/src/telemetry/otlp.rs
@@ -17,7 +17,7 @@ use super::Grant;
use crate::resolution::{ControlPlaneConfig, ResolutionError, signing};
const INGESTION_PATH: &str = "/api/public/otel/v1/traces";
-const MAX_PAYLOAD_BYTES: usize = 8 * 1024 * 1024;
+pub(super) const MAX_PAYLOAD_BYTES: usize = 20 * 1024 * 1024;
const MAX_RESPONSE_BYTES: usize = 64 * 1024;
const UPLOAD_TIMEOUT: Duration = Duration::from_secs(30);
diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rs
index fc908ec9de..0247943d2a 100644
--- a/ai-gateway/src/telemetry/tests.rs
+++ b/ai-gateway/src/telemetry/tests.rs
@@ -8,7 +8,7 @@ use serde_json::{Value, json};
use tokio::sync::Notify;
use super::{
- otlp::{ExportError, Payload},
+ otlp::{self, ExportError, Payload},
*,
};
use crate::{
@@ -292,7 +292,7 @@ async fn expired_grants_and_oversized_payloads_never_reach_the_network() {
uploader.export(&expired.grant, &empty_payload()).await,
Err(ExportError::Expired)
);
- let oversized = json!({"oversized": "x".repeat(8 * 1024 * 1024)});
+ let oversized = json!({"oversized": "x".repeat(otlp::MAX_PAYLOAD_BYTES)});
assert_eq!(
Payload::encode(&[oversized]).err(),
Some(ExportError::Payload)
@@ -654,6 +654,127 @@ async fn retained_byte_budget_is_released_after_successful_drain() {
assert_eq!(web.calls(), 1);
}
+#[tokio::test]
+async fn full_input_at_capture_limit_is_uploaded() {
+ let web = FakeServer::start(|_| async { response(200, "{}") }).await;
+ let telemetry = Telemetry::with_uploader(
+ uploader(&web.url),
+ 1,
+ DEFAULT_RETAINED_BYTES,
+ BatchPolicy::default(),
+ fast_retry(),
+ );
+ let mut large = facts("project-1");
+ large.metadata["ingestion_mode"] = json!("full");
+ let quotes = (crate::capture::MAX_INPUT_CAPTURE_BYTES - 16) / 2;
+ let input = json!({"input": "\"".repeat(quotes)});
+ assert!(input.to_string().len() <= crate::capture::MAX_INPUT_CAPTURE_BYTES);
+ large.inference.input = Some(input);
+ large.inference.output = Some(json!([
+ "x".repeat(crate::capture::MAX_OUTPUT_CAPTURE_BYTES - 16)
+ ]));
+ telemetry.record(grant().await, large);
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(5))
+ .await;
+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 0);
+ assert_eq!(telemetry.0.stats.accepted.load(Ordering::Relaxed), 1);
+ assert_eq!(web.calls(), 1);
+}
+
+#[tokio::test]
+async fn input_that_does_not_fit_is_omitted_instead_of_dropping_the_generation() {
+ let input_size = crate::capture::MAX_INPUT_CAPTURE_BYTES - 16;
+ // The configured buffer minimum is smaller than a record carrying this input.
+ for (buffer, held, reason) in [
+ (4 * 1024 * 1024, 0, "record_limit"),
+ (
+ DEFAULT_RETAINED_BYTES,
+ DEFAULT_RETAINED_BYTES - 1024 * 1024,
+ "telemetry_buffer",
+ ),
+ ] {
+ let received = Arc::new(Mutex::new(Vec::new()));
+ let sink = received.clone();
+ let web = FakeServer::start(move |request| {
+ let sink = sink.clone();
+ async move {
+ let bytes = to_bytes(request.into_body(), usize::MAX).await.unwrap();
+ let payload = upload_json(&bytes);
+ let attributes: Map<String, Value> =
+ payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|attribute| {
+ (
+ attribute["key"].as_str().unwrap().to_owned(),
+ attribute["value"]["stringValue"].clone(),
+ )
+ })
+ .collect();
+ sink.lock().unwrap().push(attributes);
+ response(200, "{}")
+ }
+ })
+ .await;
+ let telemetry = Telemetry::with_uploader(
+ uploader(&web.url),
+ 1,
+ buffer,
+ BatchPolicy::default(),
+ fast_retry(),
+ );
+ let held = (held > 0).then(|| {
+ telemetry
+ .0
+ .retained
+ .clone()
+ .try_acquire_many_owned(u32::try_from(held).unwrap())
+ .unwrap()
+ });
+ let mut large = facts("project-1");
+ large.metadata["ingestion_mode"] = json!("full");
+ large.inference.input = Some(json!("x".repeat(input_size)));
+ large.inference.output = Some(json!(["output-canary"]));
+ telemetry.record(grant().await, large);
+ drop(held);
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(5))
+ .await;
+ assert_eq!(
+ telemetry.0.stats.dropped.load(Ordering::Relaxed),
+ 0,
+ "{reason}"
+ );
+ assert_eq!(
+ telemetry.0.stats.accepted.load(Ordering::Relaxed),
+ 1,
+ "{reason}"
+ );
+ let received = received.lock().unwrap();
+ let attributes = &received[0];
+ assert!(!attributes.contains_key("langfuse.observation.input"));
+ assert!(
+ attributes["langfuse.observation.output"]
+ .as_str()
+ .unwrap()
+ .contains("output-canary")
+ );
+ let metadata: Value = serde_json::from_str(
+ attributes["langfuse.observation.metadata"]
+ .as_str()
+ .unwrap(),
+ )
+ .unwrap();
+ assert_eq!(metadata["langfuse.gateway.request.input_omitted"], reason);
+ assert_eq!(
+ metadata["langfuse.gateway.request.input_bytes"],
+ input_size + 2
+ );
+ }
+}
+
#[tokio::test]
async fn oversized_records_are_dropped_before_admission() {
let web = FakeServer::start(|_| async { response(200, "{}") }).await;