Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ai-gateway): keep captured request JSON raw to bound trace memory (#19576)

Tobias Wochinger59c0fb8d25ed
diff --git a/ai-gateway/src/capture/anthropic_messages.rs b/ai-gateway/src/capture/anthropic_messages.rs
index 6223716fab..16170a902d 100644
--- a/ai-gateway/src/capture/anthropic_messages.rs
+++ b/ai-gateway/src/capture/anthropic_messages.rs
@@ -1,6 +1,6 @@
use super::{
- MAX_FACT_STRING, MAX_ITEMS, MAX_OUTPUT_CAPTURE_BYTES, bounded_string, facts::ProviderFacts,
- parse_request, response::ResponseBody,
+ MAX_FACT_STRING, MAX_ITEMS, MAX_OUTPUT_CAPTURE_BYTES, RawRequest, bounded_string,
+ facts::ProviderFacts, parse_request, response::ResponseBody,
};
use crate::resolution::IngestionMode;
use axum::http::HeaderMap;
@@ -289,14 +289,14 @@ impl AnthropicMessagesCapture {
self.facts
}
- fn capture_request(&mut self, mut request: Map<String, Value>) {
+ fn capture_request(&mut self, mut request: RawRequest) {
self.facts.requested_model = request
- .remove("model")
+ .take("model")
.as_ref()
.and_then(Value::as_str)
.and_then(bounded_string);
for key in SCALAR_PARAMETERS {
- if let Some(value) = request.remove(key).filter(|v| {
+ if let Some(value) = request.take(key).filter(|v| {
v.is_number()
|| v.is_boolean()
|| v.as_str().is_some_and(|s| s.len() <= MAX_FACT_STRING)
@@ -306,20 +306,20 @@ impl AnthropicMessagesCapture {
}
if self.mode == IngestionMode::Full {
for key in STRUCTURED_PARAMETERS {
- if let Some(value) = request.remove(key) {
+ if let Some(value) = request.take(key) {
self.facts.model_parameters.insert(key.to_owned(), value);
}
}
// Claude Code stores its session identifiers here; it is request
// metadata rather than prompt content.
- if let Some(value) = request.remove("metadata") {
+ if let Some(value) = request.take("metadata") {
self.facts
.request_metadata
.insert("metadata".to_owned(), value);
}
// `system`, `messages`, `tools` and unknown fields remain native.
- self.facts.input = Some(Value::Object(request));
- self.facts.input_complete = true;
+ self.facts.input = request.into_input();
+ self.facts.input_complete = self.facts.input.is_some();
}
}
diff --git a/ai-gateway/src/capture/anthropic_messages/tests.rs b/ai-gateway/src/capture/anthropic_messages/tests.rs
index f5273d6495..ae4adec9d2 100644
--- a/ai-gateway/src/capture/anthropic_messages/tests.rs
+++ b/ai-gateway/src/capture/anthropic_messages/tests.rs
@@ -291,7 +291,7 @@ async fn request_configuration_and_metadata_are_projected_out_of_input_by_mode()
.cloned()
.unwrap()
);
- let input = facts.input.as_ref().unwrap();
+ let input = facts.input_json().unwrap();
assert_eq!(input["system"][0]["text"], json!("system-canary"));
assert_eq!(
input["system"][0]["cache_control"]["type"],
diff --git a/ai-gateway/src/capture/facts.rs b/ai-gateway/src/capture/facts.rs
index ed4a02f98c..b454146b3f 100644
--- a/ai-gateway/src/capture/facts.rs
+++ b/ai-gateway/src/capture/facts.rs
@@ -1,6 +1,6 @@
//! Owned facts at the capture/telemetry boundary. Content and usage remain native JSON.
use serde::Serialize;
-use serde_json::{Map, Value};
+use serde_json::{Map, Value, value::RawValue};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
@@ -70,7 +70,7 @@ pub(crate) struct ProviderFacts {
pub request_metadata: Map<String, Value>,
pub error_message: Option<String>,
pub usage_details: Option<Value>,
- pub input: Option<Value>,
+ pub input: Option<Box<RawValue>>,
pub input_omission: Option<InputOmission>,
pub output: Option<Value>,
pub provider_response_id: Option<String>,
@@ -80,3 +80,12 @@ pub(crate) struct ProviderFacts {
pub output_complete: bool,
pub capture_complete: bool,
}
+
+#[cfg(test)]
+impl ProviderFacts {
+ pub(crate) fn input_json(&self) -> Option<Value> {
+ self.input
+ .as_ref()
+ .map(|raw| serde_json::from_str(raw.get()).unwrap())
+ }
+}
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rs
index fe6160e145..cf06213632 100644
--- a/ai-gateway/src/capture/mod.rs
+++ b/ai-gateway/src/capture/mod.rs
@@ -5,10 +5,13 @@ mod openai_responses;
mod response;
mod sse;
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::{
+ collections::BTreeMap,
+ time::{SystemTime, UNIX_EPOCH},
+};
use axum::http::{HeaderMap, header};
-use serde_json::{Map, Value, json};
+use serde_json::{Map, Value, json, value::RawValue};
use tokio::time::Instant;
use crate::{
@@ -26,6 +29,7 @@ pub(crate) const MAX_INPUT_CAPTURE_BYTES: usize = 5 * 1024 * 1024;
pub(crate) const MAX_OUTPUT_CAPTURE_BYTES: usize = 1024 * 1024;
const MAX_ITEMS: usize = 256;
const MAX_FACT_STRING: usize = 512;
+const MAX_PROJECTED_BYTES: usize = 64 * 1024;
enum ProtocolCapture {
OpenAiResponses(OpenAiResponsesCapture),
@@ -288,13 +292,51 @@ impl Drop for ExecutionCapture {
}
}
-fn parse_request(headers: &HeaderMap, body: &[u8]) -> Result<Map<String, Value>, InputOmission> {
+/// The request's top-level members, each kept as its original bytes. A parsed
+/// `Value` tree can take about 100x the bytes of adversarial JSON such as
+/// `[{"":0},...]`, so the members projected out of the input share one small
+/// parse budget.
+struct RawRequest {
+ members: BTreeMap<String, Box<RawValue>>,
+ parse_budget: usize,
+}
+
+impl RawRequest {
+ /// Remove and parse a member that fits the remaining parse budget. Members
+ /// that do not fit stay in the request and so in the recorded input.
+ fn take(&mut self, key: &str) -> Option<Value> {
+ self.take_where(key, |raw| serde_json::from_str(raw.get()).ok())
+ }
+
+ /// Remove a member that fits the remaining parse budget and that `parse` accepts.
+ fn take_where<T>(
+ &mut self,
+ key: &str,
+ parse: impl FnOnce(&RawValue) -> Option<T>,
+ ) -> Option<T> {
+ let raw = self.members.get(key)?;
+ let remaining = self.parse_budget.checked_sub(raw.get().len())?;
+ let value = parse(raw)?;
+ self.parse_budget = remaining;
+ self.members.remove(key);
+ Some(value)
+ }
+
+ fn into_input(self) -> Option<Box<RawValue>> {
+ serde_json::value::to_raw_value(&self.members).ok()
+ }
+}
+
+fn parse_request(headers: &HeaderMap, body: &[u8]) -> Result<RawRequest, InputOmission> {
let reason = if !identity_encoding(headers) {
InputOmissionReason::ContentEncoding
} else if body.len() > MAX_INPUT_CAPTURE_BYTES {
InputOmissionReason::SizeLimit
- } else if let Ok(Value::Object(request)) = serde_json::from_slice(body) {
- return Ok(request);
+ } else if let Ok(members) = serde_json::from_slice(body) {
+ return Ok(RawRequest {
+ members,
+ parse_budget: MAX_PROJECTED_BYTES,
+ });
} else {
InputOmissionReason::InvalidJson
};
diff --git a/ai-gateway/src/capture/openai_chat_completions.rs b/ai-gateway/src/capture/openai_chat_completions.rs
index 5969489e31..b7fed63069 100644
--- a/ai-gateway/src/capture/openai_chat_completions.rs
+++ b/ai-gateway/src/capture/openai_chat_completions.rs
@@ -1,7 +1,7 @@
//! `OpenAI` Chat Completions request, JSON response and streamed choice capture.
use super::{
- MAX_FACT_STRING, MAX_ITEMS, MAX_OUTPUT_CAPTURE_BYTES, bounded_string, facts::ProviderFacts,
- parse_request, response::ResponseBody,
+ MAX_FACT_STRING, MAX_ITEMS, MAX_OUTPUT_CAPTURE_BYTES, RawRequest, bounded_string,
+ facts::ProviderFacts, parse_request, response::ResponseBody,
};
use crate::resolution::IngestionMode;
use axum::http::HeaderMap;
@@ -349,14 +349,14 @@ impl OpenAiChatCompletionsCapture {
self.facts
}
- fn capture_request(&mut self, mut request: Map<String, Value>) {
+ fn capture_request(&mut self, mut request: RawRequest) {
self.facts.requested_model = request
- .remove("model")
+ .take("model")
.as_ref()
.and_then(Value::as_str)
.and_then(bounded_string);
for key in SCALAR_PARAMETERS {
- if let Some(value) = request.remove(key).filter(|v| {
+ if let Some(value) = request.take(key).filter(|v| {
self.mode == IngestionMode::Full
|| v.is_number()
|| v.is_boolean()
@@ -367,18 +367,18 @@ impl OpenAiChatCompletionsCapture {
}
if self.mode == IngestionMode::Full {
for key in STRUCTURED_PARAMETERS {
- if let Some(value) = request.remove(key) {
+ if let Some(value) = request.take(key) {
self.facts.model_parameters.insert(key.to_owned(), value);
}
}
for key in REQUEST_METADATA {
- if let Some(value) = request.remove(key) {
+ if let Some(value) = request.take(key) {
self.facts.request_metadata.insert(key.to_owned(), value);
}
}
// `messages`, `tools`, legacy `functions` and unknown fields remain native.
- self.facts.input = Some(Value::Object(request));
- self.facts.input_complete = true;
+ self.facts.input = request.into_input();
+ self.facts.input_complete = self.facts.input.is_some();
}
}
diff --git a/ai-gateway/src/capture/openai_chat_completions/tests.rs b/ai-gateway/src/capture/openai_chat_completions/tests.rs
index ac9e7d8352..03098a5625 100644
--- a/ai-gateway/src/capture/openai_chat_completions/tests.rs
+++ b/ai-gateway/src/capture/openai_chat_completions/tests.rs
@@ -186,7 +186,7 @@ async fn request_configuration_and_metadata_are_projected_out_of_input() {
let mut observer = new_observer("full").await;
let (_, facts) = finalized(&mut observer);
assert_eq!(
- facts.input,
+ facts.input_json(),
Some(json!({
"messages": [{"role":"system","content":"system-canary"},{"role":"user","content":"prompt-canary"}],
"tools": [{"type":"function","function":{"name":"lookup","parameters":{"type":"object"}}}],
diff --git a/ai-gateway/src/capture/openai_responses.rs b/ai-gateway/src/capture/openai_responses.rs
index c170629bbb..3dd48c7140 100644
--- a/ai-gateway/src/capture/openai_responses.rs
+++ b/ai-gateway/src/capture/openai_responses.rs
@@ -1,7 +1,7 @@
//! `OpenAI` Responses request, JSON response and completed SSE item capture.
use super::{
- MAX_FACT_STRING, MAX_ITEMS, MAX_OUTPUT_CAPTURE_BYTES, bounded_string, facts::ProviderFacts,
- parse_request, response::ResponseBody,
+ MAX_FACT_STRING, MAX_ITEMS, MAX_OUTPUT_CAPTURE_BYTES, RawRequest, bounded_string,
+ facts::ProviderFacts, parse_request, response::ResponseBody,
};
use crate::{resolution::IngestionMode, telemetry};
use axum::http::HeaderMap;
@@ -115,14 +115,15 @@ impl OpenAiResponsesCapture {
self.client_metadata.as_ref()
}
- fn capture_request(&mut self, mut request: Map<String, Value>) {
+ fn capture_request(&mut self, mut request: RawRequest) {
self.facts.requested_model = request
- .get("model")
+ .take("model")
+ .as_ref()
.and_then(Value::as_str)
.and_then(bounded_string);
- request.remove("model");
// Agent identifiers are recorded as `agent.*` metadata in both ingestion modes.
- self.client_metadata = telemetry::take_agent_client_metadata(&mut request);
+ self.client_metadata =
+ request.take_where("client_metadata", telemetry::agent_client_metadata);
for key in [
"temperature",
"top_p",
@@ -137,7 +138,7 @@ impl OpenAiResponsesCapture {
"store",
"prompt_cache_retention",
] {
- if let Some(value) = request.remove(key).filter(|v| {
+ if let Some(value) = request.take(key).filter(|v| {
self.mode == IngestionMode::Full
|| v.is_number()
|| v.is_boolean()
@@ -157,18 +158,18 @@ impl OpenAiResponsesCapture {
"moderation",
"prompt_cache_options",
] {
- if let Some(value) = request.remove(key) {
+ if let Some(value) = request.take(key) {
self.facts.model_parameters.insert(key.to_owned(), value);
}
}
for key in ["metadata", "prompt_cache_key", "safety_identifier", "user"] {
- if let Some(value) = request.remove(key) {
+ if let Some(value) = request.take(key) {
self.facts.request_metadata.insert(key.to_owned(), value);
}
}
// Preserve native prompt context and unknown fields after projecting configuration.
- self.facts.input = Some(Value::Object(request));
- self.facts.input_complete = true;
+ self.facts.input = request.into_input();
+ self.facts.input_complete = self.facts.input.is_some();
}
}
diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rs
index 72051b9f25..ea9586b426 100644
--- a/ai-gateway/src/capture/openai_responses/tests.rs
+++ b/ai-gateway/src/capture/openai_responses/tests.rs
@@ -96,7 +96,7 @@ async fn captures_completed_items_only_in_order_without_duplication() {
40
);
assert_eq!(
- capture.facts.input.as_ref().unwrap()["instructions"],
+ capture.facts.input_json().unwrap()["instructions"],
"prompt-canary"
);
assert!(capture.facts.output_complete);
@@ -206,7 +206,7 @@ async fn request_configuration_is_projected_out_of_input() {
)
.into_facts();
assert_eq!(
- capture.input.unwrap(),
+ capture.input_json().unwrap(),
json!({
"input": "hello", "instructions": "be brief",
"tools": [{"type": "function", "name": "weather"}],
@@ -225,6 +225,34 @@ async fn request_configuration_is_projected_out_of_input() {
assert_eq!(capture.request_metadata["prompt_cache_key"], "cache-key");
}
+#[tokio::test]
+async fn configuration_beyond_the_parse_budget_stays_in_input() {
+ // Each member fits the budget on its own, but not together.
+ let half = json!({"schema": vec![json!({"": 0}); 5 * 1024]});
+ let request = json!({
+ "model": "requested", "input": "hello",
+ "text": half, "metadata": half, "temperature": 0.5,
+ });
+ let capture = OpenAiResponsesCapture::new(
+ &HeaderMap::new(),
+ request.to_string().as_bytes(),
+ IngestionMode::Full,
+ )
+ .into_facts();
+ assert_eq!(
+ capture.input_json().unwrap(),
+ json!({"input": "hello", "metadata": half})
+ );
+ assert_eq!(
+ capture.model_parameters,
+ json!({"temperature": 0.5, "text": half})
+ .as_object()
+ .unwrap()
+ .clone()
+ );
+ assert!(capture.request_metadata.is_empty());
+}
+
#[tokio::test]
async fn agent_client_metadata_is_projected_out_of_input_in_both_modes() {
let codex = json!({
@@ -245,7 +273,7 @@ async fn agent_client_metadata_is_projected_out_of_input_in_both_modes() {
let facts = capture.into_facts();
if mode == IngestionMode::Full {
assert_eq!(
- facts.input.unwrap(),
+ facts.input_json().unwrap(),
json!({"input": "hello", "future_field": {"keep": true}})
);
} else {
@@ -264,7 +292,7 @@ async fn agent_client_metadata_is_projected_out_of_input_in_both_modes() {
);
assert!(capture.client_metadata().is_none());
assert_eq!(
- capture.into_facts().input.unwrap(),
+ capture.into_facts().input_json().unwrap(),
json!({"input": "hello", "client_metadata": {"team": "search"}})
);
}
@@ -923,10 +951,7 @@ async fn request_at_input_limit_is_captured_in_full() {
assert!(facts.input_complete);
assert!(facts.input_omission.is_none());
assert_eq!(
- facts.input.as_ref().unwrap()["input"]
- .as_str()
- .unwrap()
- .len(),
+ facts.input_json().unwrap()["input"].as_str().unwrap().len(),
MAX_INPUT_CAPTURE_BYTES - 12
);
}
@@ -968,7 +993,7 @@ async fn capture_regression_preserves_native_request_and_usage() {
let mut expected_input = request.clone();
expected_input.as_object_mut().unwrap().remove("model");
assert_eq!(
- facts.input.as_ref(),
+ facts.input_json().as_ref(),
if mode == "full" {
Some(&expected_input)
} else {
diff --git a/ai-gateway/src/providers/chat_completions.rs b/ai-gateway/src/providers/chat_completions.rs
index 1f6d4736bf..b32622c727 100644
--- a/ai-gateway/src/providers/chat_completions.rs
+++ b/ai-gateway/src/providers/chat_completions.rs
@@ -7,7 +7,7 @@ use serde::{
Deserialize, Deserializer,
de::{MapAccess, Visitor},
};
-use serde_json::{Value, value::RawValue};
+use serde_json::value::RawValue;
/// The rewritten body for a streamed request that did not ask for usage, or
/// `None` when the original bytes must be forwarded unchanged. Every other
@@ -24,38 +24,43 @@ pub(super) fn request_stream_usage(body: &[u8], provider_model: &str) -> Option<
if member(&members, "stream")? != "true" {
return None;
}
- let options = match member(&members, "stream_options") {
- None => serde_json::Map::new(),
- Some(raw) => match serde_json::from_str::<Value>(raw).ok()? {
- Value::Object(options) => options,
- Value::Null => serde_json::Map::new(),
- _ => return None,
- },
+ // Parsed as raw members too: a full `Value` tree can take ~100x the bytes.
+ let mut options = match member(&members, "stream_options") {
+ None | Some("null") => Vec::new(),
+ Some(raw) => serde_json::from_str::<Members>(raw).ok()?.0,
};
- if options.get("include_usage") == Some(&Value::Bool(true)) {
+ if count(&options, "include_usage") > 1 || member(&options, "include_usage") == Some("true") {
return None;
}
- let mut options = options;
- options.insert("include_usage".to_owned(), Value::Bool(true));
- let options = RawValue::from_string(Value::Object(options).to_string()).ok()?;
+ let include_usage = RawValue::from_string("true".to_owned()).ok()?;
+ match options.iter_mut().find(|(key, _)| key == "include_usage") {
+ Some((_, raw)) => *raw = include_usage,
+ None => options.push(("include_usage".to_owned(), include_usage)),
+ }
+ let options =
+ RawValue::from_string(String::from_utf8(object_bytes(&options, 32)?).ok()?).ok()?;
let model = RawValue::from_string(serde_json::to_string(provider_model).ok()?).ok()?;
members.iter_mut().find(|(key, _)| key == "model")?.1 = model;
match members.iter_mut().find(|(key, _)| key == "stream_options") {
Some((_, raw)) => *raw = options,
None => members.push(("stream_options".to_owned(), options)),
}
- let mut rewritten = Vec::with_capacity(body.len() + 32);
- rewritten.push(b'{');
+ object_bytes(&members, body.len() + 32).map(Bytes::from)
+}
+
+fn object_bytes(members: &[(String, Box<RawValue>)], capacity: usize) -> Option<Vec<u8>> {
+ let mut bytes = Vec::with_capacity(capacity);
+ bytes.push(b'{');
for (index, (key, raw)) in members.iter().enumerate() {
if index > 0 {
- rewritten.push(b',');
+ bytes.push(b',');
}
- serde_json::to_writer(&mut rewritten, key).ok()?;
- rewritten.push(b':');
- rewritten.extend_from_slice(raw.get().as_bytes());
+ serde_json::to_writer(&mut bytes, key).ok()?;
+ bytes.push(b':');
+ bytes.extend_from_slice(raw.get().as_bytes());
}
- rewritten.push(b'}');
- Some(Bytes::from(rewritten))
+ bytes.push(b'}');
+ Some(bytes)
}
fn count(members: &[(String, Box<RawValue>)], name: &str) -> usize {
diff --git a/ai-gateway/src/telemetry/context.rs b/ai-gateway/src/telemetry/context.rs
index bf2f27c417..677a45fdf6 100644
--- a/ai-gateway/src/telemetry/context.rs
+++ b/ai-gateway/src/telemetry/context.rs
@@ -9,7 +9,7 @@ use opentelemetry_sdk::{
trace::{IdGenerator, RandomIdGenerator},
};
use serde::Serialize;
-use serde_json::{Map, Value};
+use serde_json::{Map, Value, value::RawValue};
use sha2::{Digest, Sha256};
const MAX_HEADER_BYTES: usize = 8 * 1024;
@@ -127,7 +127,7 @@ impl GenerationContext {
}
/// `client_metadata` is the request body's agent metadata object, already
- /// removed from the captured input by [`take_agent_client_metadata`].
+ /// removed from the captured input by [`agent_client_metadata`].
pub(super) fn from_request(
headers: &HeaderMap,
client_metadata: Option<&Map<String, Value>>,
@@ -429,27 +429,16 @@ impl AgentContext {
}
}
-/// Remove a coding agent's `client_metadata` object from a captured request so its
-/// identifiers are recorded as `agent.*` metadata rather than as prompt input.
-/// Unrecognized or oversized objects stay in the request untouched.
-pub(crate) fn take_agent_client_metadata(
- request: &mut Map<String, Value>,
-) -> Option<Map<String, Value>> {
- let recognized = request
- .get("client_metadata")
- .and_then(Value::as_object)
- .is_some_and(|metadata| {
- is_codex_client_metadata(metadata)
- && serde_json::to_vec(metadata)
- .is_ok_and(|bytes| bytes.len() <= MAX_CLIENT_METADATA_BYTES)
- });
- if !recognized {
+/// A coding agent's `client_metadata` object, parsed so its identifiers are recorded
+/// as `agent.*` metadata rather than as prompt input. Unrecognized or oversized
+/// objects yield `None` and stay in the captured request untouched.
+pub(crate) fn agent_client_metadata(raw: &RawValue) -> Option<Map<String, Value>> {
+ if raw.get().len() > MAX_CLIENT_METADATA_BYTES {
return None;
}
- match request.remove("client_metadata") {
- Some(Value::Object(metadata)) => Some(metadata),
- _ => None,
- }
+ serde_json::from_str(raw.get())
+ .ok()
+ .filter(is_codex_client_metadata)
}
fn is_codex_client_metadata(metadata: &Map<String, Value>) -> bool {
diff --git a/ai-gateway/src/telemetry/context/tests.rs b/ai-gateway/src/telemetry/context/tests.rs
index 00da523fd8..6699ed0e7d 100644
--- a/ai-gateway/src/telemetry/context/tests.rs
+++ b/ai-gateway/src/telemetry/context/tests.rs
@@ -479,33 +479,17 @@ fn codex_body_snapshot_is_preferred_and_flat_keys_fill_missing_snapshots() {
#[test]
fn agent_client_metadata_is_taken_only_when_recognized_and_bounded() {
- let mut request: Map<String, Value> = json!({
- "input": "hello",
- "client_metadata": codex_client_metadata(Some(&codex_turn_metadata())),
- })
- .as_object()
- .unwrap()
- .clone();
- let taken = take_agent_client_metadata(&mut request).unwrap();
+ let raw = |value: Value| serde_json::value::to_raw_value(&value).unwrap();
+ let metadata = Value::Object(codex_client_metadata(Some(&codex_turn_metadata())));
+ let taken = agent_client_metadata(&raw(metadata)).unwrap();
assert_eq!(taken["x-codex-installation-id"], "install-1");
- assert_eq!(
- request,
- json!({"input": "hello"}).as_object().unwrap().clone()
- );
for client_metadata in [
json!({"team": "search"}),
json!("x-codex-turn-metadata"),
json!({"x-codex-turn-metadata": "a".repeat(MAX_CLIENT_METADATA_BYTES)}),
] {
- let mut request: Map<String, Value> =
- json!({"input": "hello", "client_metadata": client_metadata})
- .as_object()
- .unwrap()
- .clone();
- let before = request.clone();
- assert!(take_agent_client_metadata(&mut request).is_none());
- assert_eq!(request, before);
+ assert!(agent_client_metadata(&raw(client_metadata)).is_none());
}
}
diff --git a/ai-gateway/src/telemetry/mapping.rs b/ai-gateway/src/telemetry/mapping.rs
index 8ec338afe5..ff75514341 100644
--- a/ai-gateway/src/telemetry/mapping.rs
+++ b/ai-gateway/src/telemetry/mapping.rs
@@ -450,7 +450,10 @@ mod tests {
.inference
.model_parameters
.insert("service_tier".into(), json!("priority"));
- facts.inference.input = Some(json!({"input": "hello", "tools": [{"type": "function"}]}));
+ facts.inference.input = serde_json::value::to_raw_value(
+ &json!({"input": "hello", "tools": [{"type": "function"}]}),
+ )
+ .ok();
facts.inference.output = Some(json!([{ "type": "message", "content": "world" }]));
facts.inference.capture_complete = true;
facts.inference.output_complete = true;
diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rs
index 6572064990..c96c093e6e 100644
--- a/ai-gateway/src/telemetry/mod.rs
+++ b/ai-gateway/src/telemetry/mod.rs
@@ -31,7 +31,7 @@ use crate::{
};
use batch::{BatchPolicy, Batches, Pending};
use context::GenerationContext;
-pub(crate) use context::take_agent_client_metadata;
+pub(crate) use context::agent_client_metadata;
use otlp::Uploader;
use retry::RetryPolicy;
use secrecy::{ExposeSecret, SecretString};
diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rs
index 50d48f36f3..1ecd6d3ff3 100644
--- a/ai-gateway/src/telemetry/tests.rs
+++ b/ai-gateway/src/telemetry/tests.rs
@@ -685,7 +685,7 @@ async fn full_input_at_capture_limit_is_uploaded() {
let quotes = (crate::capture::MAX_INPUT_CAPTURE_BYTES - 16) / 2;
let input = json!({"input": "\"".repeat(quotes)});
assert!(input.to_string().len() <= crate::capture::MAX_INPUT_CAPTURE_BYTES);
- large.inference.input = Some(input);
+ large.inference.input = serde_json::value::to_raw_value(&input).ok();
large.inference.output = Some(json!([
"x".repeat(crate::capture::MAX_OUTPUT_CAPTURE_BYTES - 16)
]));
@@ -751,7 +751,7 @@ async fn input_that_does_not_fit_is_omitted_instead_of_dropping_the_generation()
});
let mut large = facts("project-1");
large.metadata["ingestion_mode"] = json!("full");
- large.inference.input = Some(json!("x".repeat(input_size)));
+ large.inference.input = serde_json::value::to_raw_value(&"x".repeat(input_size)).ok();
large.inference.output = Some(json!(["output-canary"]));
telemetry.record(grant().await, large);
drop(held);