Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(ai-gateway)!: organize generation metadata namespaces (#17645)

Hassieb Pakzade7b88c98f0c1
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index da50000756..8006c4a59e 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -266,14 +266,34 @@ and streams without a captured content delta have no completion-start time or TT
The exporter projects native OpenAI usage into the receiver's supported shape for
pricing without duplicating it in metadata. Missing usage is not reported as zero.
-Gateway metadata uses `langfuse.gateway.*`: `project_id`, `organization_id`,
-`ingestion_mode`, `api_format`, `api-key.id`, and `provider.connection_id`,
-`provider.request.id`, `provider.response.id`. API-key attribution entries appear
-both as top-level metadata and under `langfuse.gateway.api-key.metadata.*`.
+Gateway metadata uses `langfuse.gateway.*`, grouped by the resource or exchange
+each field describes:
+
+| Suffix | Meaning |
+| ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- |
+| `organization.id`, `project.id` | Resolved Langfuse organization and ingestion project |
+| `api_key.id`, `connection.id` | Authenticated gateway key and selected Langfuse connection |
+| `ingestion.mode` | Effective capture mode |
+| `request.api_format` | Native API contract, such as `openai.responses` |
+| `request.metadata`, `request.prompt_cache_key`, `request.safety_identifier`, `request.user` | Native caller-supplied fields, captured only in full mode |
+| `response.id` | Native response object's ID |
+| `response.status_code` | HTTP status returned to the caller |
+| `upstream.request.id` | Provider request ID received in the upstream `x-request-id` response header |
+
+`request.id` is reserved for a future gateway-generated request ID and is not
+emitted. The upstream request ID is separate and is omitted when unavailable.
+API-key attribution entries appear both as top-level metadata and under
+`langfuse.gateway.api_key.metadata.*`.
Gateway, agent and OpenTelemetry fields win collisions; the namespaced attribution
-copy preserves the original value. `http_status` stays top level. Relay outcome,
-provider status, completeness flags and first-byte timing remain internal facts
-rather than generation metadata. Ingestion removes mapped observation-attribute
+copy preserves the original value. `agent.*` and native request field names retain
+their existing spelling. Metadata names apply to newly captured generations;
+historical traces are not rewritten.
+
+`response.status_code` includes gateway-generated 502/504 errors before upstream
+headers arrive. After headers arrive it retains the relayed status, even if a
+stream subsequently fails; cancellation before headers leaves the status unknown.
+Relay outcome, provider status, completeness flags and first-byte timing remain
+internal facts rather than generation metadata. Ingestion removes mapped observation-attribute
duplicates for the gateway scope while preserving custom attributes, scope and
resources.
@@ -432,7 +452,7 @@ JSON types; ingestion applies Langfuse's existing model-parameter normalization
are not filled with assumed defaults.
The response model and service tier take precedence over requested values when present.
Request `metadata`, `prompt_cache_key`, `safety_identifier` and deprecated `user` are
-stored under `langfuse.gateway.provider.request.*` only in full mode.
+stored under `langfuse.gateway.request.*` only in full mode.
In both modes, `usage_details` preserves the provider's entire usage object,
including nested and unknown fields. The gateway does not rename counters,
diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rs
index 3ec0af0fa9..9b878c2ba0 100644
--- a/ai-gateway/src/capture/openai_responses/tests.rs
+++ b/ai-gateway/src/capture/openai_responses/tests.rs
@@ -751,7 +751,7 @@ async fn client_compression_preferences_do_not_disable_capture() {
assert!(metadata.get("output_complete").is_none());
assert!(metadata.get("provider_status").is_none());
assert!(metadata.get("native_usage").is_none());
- assert_eq!(metadata["langfuse.gateway.provider.response.id"], "resp-1");
+ assert_eq!(metadata["langfuse.gateway.response.id"], "resp-1");
assert!(
uploaded_attribute(&upload, "langfuse.observation.usage_details")["input_tokens"]
.is_number()
@@ -843,7 +843,7 @@ async fn codex_body_metadata_reaches_the_generation_without_agent_headers() {
assert_eq!(metadata["agent.request_kind"], "turn");
assert_eq!(metadata["agent.sandbox_mode"], "workspace-write");
assert!(metadata.get("agent.tool_namespaces_info").is_none());
- assert_eq!(metadata["langfuse.gateway.provider.response.id"], "resp-1");
+ assert_eq!(metadata["langfuse.gateway.response.id"], "resp-1");
assert_eq!(
uploaded_attribute(&upload, "langfuse.observation.input"),
json!({"input": "hello"})
diff --git a/ai-gateway/src/http/tests.rs b/ai-gateway/src/http/tests.rs
index 9a0e9401e0..1251f06971 100644
--- a/ai-gateway/src/http/tests.rs
+++ b/ai-gateway/src/http/tests.rs
@@ -498,6 +498,90 @@ async fn slow_request_body_and_resolution_have_bounded_waits() {
assert_eq!(provider.calls(), 0);
}
+#[tokio::test]
+async fn pre_header_provider_failures_export_the_status_returned_to_the_caller() {
+ use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};
+ use serde_json::Value;
+
+ for expected in [StatusCode::GATEWAY_TIMEOUT, StatusCode::BAD_GATEWAY] {
+ let (sent, mut received) = tokio::sync::mpsc::channel(1);
+ let sink = FakeServer::start(move |request| {
+ let sent = sent.clone();
+ async move {
+ let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
+ sent.send(serde_json::from_slice::<Value>(&bytes).unwrap())
+ .await
+ .unwrap();
+ Response::new(Body::from("{}"))
+ }
+ })
+ .await;
+ let telemetry =
+ Telemetry::new(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap()).unwrap();
+ let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let upstream_url = format!("http://{}", upstream.local_addr().unwrap());
+ let upstream_task = tokio::spawn(async move {
+ let (_connection, _) = upstream.accept().await.unwrap();
+ if expected == StatusCode::GATEWAY_TIMEOUT {
+ std::future::pending::<()>().await;
+ }
+ });
+ let web = FakeServer::start(|_| async { resolution_response("provider-secret") }).await;
+ let provider = OpenAiProvider::for_test(
+ upstream_url,
+ ProviderLimits {
+ headers_timeout: Duration::from_millis(100),
+ ..ProviderLimits::default()
+ },
+ )
+ .with_telemetry(telemetry.clone());
+ let gateway = Gateway::start(Some(InferenceService::for_test(
+ web.control_plane(),
+ provider,
+ 1,
+ )))
+ .await;
+ let response = gateway
+ .post()
+ .bearer_auth("gateway-key")
+ .body(r#"{"model":"requested","input":"hello"}"#)
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.status(), expected);
+ let payload = tokio::time::timeout(Duration::from_secs(2), received.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ let attrs = payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]
+ .as_array()
+ .unwrap();
+ let metadata: Value = serde_json::from_str(
+ attrs
+ .iter()
+ .find(|a| a["key"] == "langfuse.observation.metadata")
+ .unwrap()["value"]["stringValue"]
+ .as_str()
+ .unwrap(),
+ )
+ .unwrap();
+ assert_eq!(
+ metadata["langfuse.gateway.response.status_code"],
+ response.status().as_u16()
+ );
+ assert!(
+ metadata
+ .get("langfuse.gateway.upstream.request.id")
+ .is_none()
+ );
+ telemetry
+ .shutdown(tokio::time::Instant::now() + Duration::from_secs(1))
+ .await;
+ assert_eq!(sink.calls(), 1);
+ upstream_task.abort();
+ }
+}
+
#[tokio::test]
async fn unconfigured_gateway_is_live_but_not_ready_and_inference_is_unavailable() {
let gateway = Gateway::start(None).await;
diff --git a/ai-gateway/src/providers/openai/tests.rs b/ai-gateway/src/providers/openai/tests.rs
index 3de94dd761..854a9df365 100644
--- a/ai-gateway/src/providers/openai/tests.rs
+++ b/ai-gateway/src/providers/openai/tests.rs
@@ -398,12 +398,12 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {
};
Response::builder()
.header("content-type", content_type)
+ .header("x-request-id", "req-1")
.body(body)
.unwrap()
})
.await;
let relay = provider(&upstream, 1).with_telemetry(telemetry.clone());
- let context = resolved_request_context_with_mode("provider-secret", "full").await;
let headers = HeaderMap::from_iter([
(
axum::http::HeaderName::from_static("traceparent"),
@@ -421,7 +421,7 @@ async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {
let response = relay
.forward(
relay.try_admit().unwrap(),
- context,
+ resolved_request_context_with_mode("provider-secret", "full").await,
&headers,
Bytes::from_static(br#"{"model":"requested","input":"hello"}"#),
)
@@ -536,6 +536,7 @@ async fn cancelled_and_timed_out_executions_upload_after_provider_context_is_rel
.unwrap()["value"]["stringValue"],
if cancelled { "WARNING" } else { "ERROR" }
);
+ assert_eq!(metadata["langfuse.gateway.response.status_code"], 200);
assert!(metadata.get("relay_outcome").is_none());
telemetry
.shutdown(Instant::now() + Duration::from_secs(1))
@@ -685,7 +686,9 @@ fn assert_completed_upload(payload: &serde_json::Value, streaming: bool) {
.unwrap();
assert!(metadata.get("relay_outcome").is_none());
assert!(metadata.get("native_usage").is_none());
- assert_eq!(metadata["langfuse.gateway.provider.response.id"], "resp-1");
+ assert_eq!(metadata["langfuse.gateway.response.id"], "resp-1");
+ assert_eq!(metadata["langfuse.gateway.upstream.request.id"], "req-1");
+ assert!(metadata.get("langfuse.gateway.request.id").is_none());
let usage: Value = serde_json::from_str(
attrs
.iter()
diff --git a/ai-gateway/src/telemetry/mapping.rs b/ai-gateway/src/telemetry/mapping.rs
index 6c56dd7087..7b7f46edc5 100644
--- a/ai-gateway/src/telemetry/mapping.rs
+++ b/ai-gateway/src/telemetry/mapping.rs
@@ -94,7 +94,6 @@ pub(super) fn span(facts: InferenceFacts, context: &GenerationContext) -> Value
fn reserved_metadata(key: &str) -> bool {
[
"langfuse.gateway",
- "http_status",
"scope",
"resourceAttributes",
"attributes",
@@ -116,40 +115,47 @@ fn generation_metadata(facts: &InferenceFacts) -> Map<String, Value> {
metadata.insert(key.clone(), value.clone());
}
metadata.insert(
- format!("langfuse.gateway.api-key.metadata.{key}"),
+ format!("langfuse.gateway.api_key.metadata.{key}"),
value.clone(),
);
}
}
for (source, target) in [
- ("project_id", "project_id"),
- ("organization_id", "organization_id"),
- ("ingestion_mode", "ingestion_mode"),
- ("key_id", "api-key.id"),
- ("provider_connection_id", "provider.connection_id"),
+ ("project_id", "project.id"),
+ ("organization_id", "organization.id"),
+ ("ingestion_mode", "ingestion.mode"),
+ ("key_id", "api_key.id"),
+ ("provider_connection_id", "connection.id"),
] {
if let Some(value) = facts.metadata.get(source) {
metadata.insert(format!("langfuse.gateway.{target}"), value.clone());
}
}
metadata.insert(
- "langfuse.gateway.api_format".into(),
+ "langfuse.gateway.request.api_format".into(),
json!(facts.api_format),
);
- metadata.insert("http_status".into(), json!(facts.http_status));
+ // Before upstream headers, transport failures become gateway HTTP errors.
+ // Once headers arrive, later stream failures cannot change the HTTP status.
+ let response_status = facts.http_status.or(match facts.outcome {
+ RelayOutcome::Timeout => Some(504),
+ RelayOutcome::TransportError => Some(502),
+ RelayOutcome::Eof | RelayOutcome::Cancelled => None,
+ });
+ metadata.insert(
+ "langfuse.gateway.response.status_code".into(),
+ json!(response_status),
+ );
for (key, value) in [
("response.id", &facts.inference.provider_response_id),
- ("request.id", &facts.inference.provider_request_id),
+ ("upstream.request.id", &facts.inference.provider_request_id),
] {
if let Some(value) = value {
- metadata.insert(format!("langfuse.gateway.provider.{key}"), json!(value));
+ metadata.insert(format!("langfuse.gateway.{key}"), json!(value));
}
}
for (key, value) in &facts.inference.request_metadata {
- metadata.insert(
- format!("langfuse.gateway.provider.request.{key}"),
- value.clone(),
- );
+ metadata.insert(format!("langfuse.gateway.request.{key}"), value.clone());
}
metadata
}
@@ -287,6 +293,27 @@ mod tests {
.unwrap()
}
+ #[test]
+ fn response_status_distinguishes_failures_before_and_after_headers() {
+ for (http_status, outcome, expected) in [
+ (None, RelayOutcome::Timeout, json!(504)),
+ (None, RelayOutcome::TransportError, json!(502)),
+ (Some(200), RelayOutcome::Timeout, json!(200)),
+ (Some(200), RelayOutcome::TransportError, json!(200)),
+ (Some(429), RelayOutcome::Eof, json!(429)),
+ (None, RelayOutcome::Cancelled, Value::Null),
+ ] {
+ let mut facts = facts();
+ facts.http_status = http_status;
+ facts.outcome = outcome;
+ let metadata = metadata(&attributes(&span(facts, &context())));
+ assert_eq!(
+ metadata["langfuse.gateway.response.status_code"], expected,
+ "{outcome:?}"
+ );
+ }
+ }
+
#[test]
fn caller_attributes_and_metadata_cannot_replace_trusted_gateway_fields() {
let mut context = context();
@@ -296,11 +323,11 @@ mod tests {
}))
.unwrap();
context.metadata = serde_json::from_value(json!({
- "custom": "value", "team": "untrusted", "http_status": "fake",
- "langfuse.gateway.project_id": "wrong-project",
- "langfuse.gateway.api-key.id": "wrong-key",
- "langfuse.gateway.api-key.metadata.team": "wrong-team",
- "langfuse.gateway.provider.request.fake": "fake",
+ "custom": "value", "team": "untrusted", "langfuse.gateway.response.status_code": "fake",
+ "langfuse.gateway.project.id": "wrong-project",
+ "langfuse.gateway.api_key.id": "wrong-key",
+ "langfuse.gateway.api_key.metadata.team": "wrong-team",
+ "langfuse.gateway.request.fake": "fake",
"scope": "fake", "resourceAttributes.secret": "fake", "attributes": "fake"
}))
.unwrap();
@@ -315,15 +342,15 @@ mod tests {
let metadata = metadata(&attrs);
assert_eq!(metadata["custom"], "value");
assert_eq!(metadata["team"], "search");
- assert_eq!(metadata["http_status"], 200);
- assert_eq!(metadata["langfuse.gateway.project_id"], "project");
- assert_eq!(metadata["langfuse.gateway.api-key.id"], "key");
- assert_eq!(metadata["langfuse.gateway.api-key.metadata.team"], "search");
+ assert_eq!(metadata["langfuse.gateway.response.status_code"], 200);
+ assert_eq!(metadata["langfuse.gateway.project.id"], "project");
+ assert_eq!(metadata["langfuse.gateway.api_key.id"], "key");
+ assert_eq!(metadata["langfuse.gateway.api_key.metadata.team"], "search");
for key in [
"scope",
"resourceAttributes.secret",
"attributes",
- "langfuse.gateway.provider.request.fake",
+ "langfuse.gateway.request.fake",
] {
assert!(metadata.get(key).is_none(), "caller forged {key}");
}
@@ -355,13 +382,13 @@ mod tests {
assert_eq!(
metadata,
json!({
- "http_status": 200,
+ "langfuse.gateway.response.status_code": 200,
"team": "search",
- "langfuse.gateway.api-key.metadata.team": "search",
- "langfuse.gateway.api-key.id": "key",
- "langfuse.gateway.project_id": "project",
- "langfuse.gateway.ingestion_mode": "full",
- "langfuse.gateway.api_format": "openai.responses",
+ "langfuse.gateway.api_key.metadata.team": "search",
+ "langfuse.gateway.api_key.id": "key",
+ "langfuse.gateway.project.id": "project",
+ "langfuse.gateway.ingestion.mode": "full",
+ "langfuse.gateway.request.api_format": "openai.responses",
})
);
}
@@ -373,11 +400,11 @@ mod tests {
facts.metadata["provider_connection_id"] = json!("connection");
facts.metadata["key_metadata"] = json!({
"team": "search", "enabled": true, "cost_center": 42,
- "http_status": 500,
- "langfuse.gateway.api-key.id": "spoofed-key",
- "langfuse.gateway.provider.request.id": "spoofed-request",
+ "langfuse.gateway.response.status_code": 500,
+ "langfuse.gateway.api_key.id": "spoofed-key",
+ "langfuse.gateway.upstream.request.id": "spoofed-request",
"langfuse.gateway.future_field": "spoofed-future",
- "langfuse.gateway.api-key.metadata.team": "spoofed-team",
+ "langfuse.gateway.api_key.metadata.team": "spoofed-team",
"agent.name": "spoofed-agent",
"scope": "spoofed-scope", "scope.name": "spoofed-scope-name",
"resourceAttributes": "spoofed-resource",
@@ -393,25 +420,19 @@ mod tests {
assert_eq!(metadata["team"], "search");
assert_eq!(metadata["enabled"], true);
assert_eq!(metadata["cost_center"], 42);
- assert_eq!(metadata["langfuse.gateway.api-key.metadata.team"], "search");
+ assert_eq!(metadata["langfuse.gateway.api_key.metadata.team"], "search");
assert_eq!(
- metadata["langfuse.gateway.api-key.metadata.http_status"],
+ metadata["langfuse.gateway.api_key.metadata.langfuse.gateway.response.status_code"],
500
);
- assert_eq!(metadata["http_status"], 200);
- assert_eq!(metadata["langfuse.gateway.api-key.id"], "key");
- assert_eq!(
- metadata["langfuse.gateway.provider.connection_id"],
- "connection"
- );
- assert_eq!(
- metadata["langfuse.gateway.provider.response.id"],
- "response"
- );
- assert_eq!(metadata["langfuse.gateway.organization_id"], "org");
+ assert_eq!(metadata["langfuse.gateway.response.status_code"], 200);
+ assert_eq!(metadata["langfuse.gateway.api_key.id"], "key");
+ assert_eq!(metadata["langfuse.gateway.connection.id"], "connection");
+ assert_eq!(metadata["langfuse.gateway.response.id"], "response");
+ assert_eq!(metadata["langfuse.gateway.organization.id"], "org");
assert_eq!(metadata["agent.name"], "opencode");
assert_eq!(
- metadata["langfuse.gateway.api-key.metadata.agent.name"],
+ metadata["langfuse.gateway.api_key.metadata.agent.name"],
"spoofed-agent"
);
for key in [
@@ -424,11 +445,11 @@ mod tests {
assert!(metadata.get(key).is_none(), "{key}");
}
assert_ne!(
- metadata["langfuse.gateway.provider.request.id"],
+ metadata["langfuse.gateway.upstream.request.id"],
"spoofed-request"
);
assert_eq!(
- metadata["langfuse.gateway.api-key.metadata.langfuse.gateway.api-key.metadata.team"],
+ metadata["langfuse.gateway.api_key.metadata.langfuse.gateway.api_key.metadata.team"],
"spoofed-team"
);
}
@@ -497,21 +518,18 @@ mod tests {
let attrs = attributes(&span(facts, &context()));
let metadata = metadata(&attrs);
assert_eq!(
- metadata["langfuse.gateway.provider.request.metadata"],
+ metadata["langfuse.gateway.request.metadata"],
json!({"customer": "customer-1"})
);
assert_eq!(
- metadata["langfuse.gateway.provider.request.prompt_cache_key"],
+ metadata["langfuse.gateway.request.prompt_cache_key"],
"cache-key"
);
assert_eq!(
- metadata["langfuse.gateway.provider.request.safety_identifier"],
+ metadata["langfuse.gateway.request.safety_identifier"],
"safety-id"
);
- assert_eq!(
- metadata["langfuse.gateway.provider.request.user"],
- "legacy-user"
- );
+ assert_eq!(metadata["langfuse.gateway.request.user"], "legacy-user");
for key in ["metadata", "prompt_cache_key", "safety_identifier", "user"] {
assert!(metadata.get(key).is_none());
}
diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rs
index f0ff2351aa..edb6554665 100644
--- a/ai-gateway/src/telemetry/tests.rs
+++ b/ai-gateway/src/telemetry/tests.rs
@@ -125,7 +125,7 @@ async fn concurrent_projects_keep_their_original_grants_and_attribution() {
serde_json::from_str(metadata["value"]["stringValue"].as_str().unwrap()).unwrap();
received.lock().unwrap().push((
authorization,
- metadata["langfuse.gateway.project_id"]
+ metadata["langfuse.gateway.project.id"]
.as_str()
.unwrap()
.to_owned(),