Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): upload finalized generations to Langfuse (#17470)

Hassieb Pakzad6adfa3f75d49
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lock
index de2918a800..3a4d768c93 100644
--- a/ai-gateway/Cargo.lock
+++ b/ai-gateway/Cargo.lock
@@ -7,6 +7,7 @@ name = "ai-gateway"
version = "0.1.0"
dependencies = [
"axum",
+ "chrono",
"futures-util",
"hmac",
"http-body-util",
@@ -18,6 +19,7 @@ dependencies = [
"tower",
"tracing",
"tracing-subscriber",
+ "uuid",
]
[[package]]
@@ -26,6 +28,12 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
[[package]]
name = "axum"
version = "0.8.9"
@@ -140,6 +148,15 @@ dependencies = [
"rand_core",
]
+[[package]]
+name = "chrono"
+version = "0.4.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
+dependencies = [
+ "num-traits",
+]
+
[[package]]
name = "cmov"
version = "0.5.4"
@@ -639,6 +656,15 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -1355,6 +1381,17 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+[[package]]
+name = "uuid"
+version = "1.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce"
+dependencies = [
+ "getrandom 0.4.3",
+ "js-sys",
+ "wasm-bindgen",
+]
+
[[package]]
name = "valuable"
version = "0.1.1"
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.toml
index 316aa4f509..c56efb94eb 100644
--- a/ai-gateway/Cargo.toml
+++ b/ai-gateway/Cargo.toml
@@ -7,6 +7,7 @@ license = "MIT"
[dependencies]
axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] }
+chrono = { version = "0.4", default-features = false, features = ["std"] }
hmac = "0.13"
http-body-util = "0.1"
futures-util = "0.3"
@@ -17,6 +18,7 @@ sha2 = "0.11"
tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }
tracing = "0.1.44"
tracing-subscriber = { version = "0.3.23", default-features = false, features = ["fmt", "json"] }
+uuid = { version = "1", features = ["v4"] }
[dev-dependencies]
tokio = { version = "1.53.1", features = ["io-util"] }
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index 336e72e09e..263bfa86e7 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -3,8 +3,9 @@
A standalone Rust gateway for `POST /openai/v1/responses`. Web resolves the gateway
key to a trusted provider connection; Rust relays native JSON or SSE without
rewriting provider bytes. A bounded capture layer captures request/response facts and
-logs the finalized record at debug level. Customer telemetry is not exported
-yet. Building and testing need no real Web, database or provider credentials.
+logs the finalized record at debug level. Each finalized execution is immediately
+uploaded as a Langfuse generation through OTLP. Building and testing need no real
+Web, database or provider credentials.
## Run locally
@@ -124,7 +125,7 @@ response = client.responses.create(model="gpt-4.1-mini", input="Say hello")
print(response.output_text)
```
-The gateway forwards to the official OpenAI Responses endpoint only. It does not
+The gateway forwards inference to the official OpenAI Responses endpoint only. It does not
retry, follow redirects, or accept client routing overrides. Request parsing is
best-effort capture only; Web still selects the provider connection.
Provider errors retain their status and body. Gateway errors use an OpenAI-style
@@ -158,6 +159,50 @@ timing/version/rate-limit
headers are retained. Cookies, routing overrides, gateway/ingestion credentials,
hop-by-hop headers and upstream framing are excluded.
+## Langfuse uploads
+
+When inference is configured, each finalized execution starts one background POST
+to the Web base URL's `/api/public/otel/v1/traces` endpoint. The client response and
+provider admission never wait for ingestion. There is no batching, waiting queue,
+retry, or durable delivery. A successful upload acknowledges ingestion acceptance;
+storage and cost processing still happen asynchronously in Langfuse.
+
+The original resolver-issued project grant authenticates the upload, together with
+a fresh gateway HMAC signature over that grant. The uploader uses the existing Web
+URL and service key, preserves deployment prefixes, disables redirects and proxies,
+and never sends the provider credential or the client's gateway key to ingestion.
+Ingestion JWTs stay outside serializable capture facts and debug logs. Expired grants
+are dropped without re-resolving the execution. Requests explicitly select v4 ingestion.
+
+Each execution becomes one root generation with generated trace/observation IDs,
+actual/requested model, model parameters, native usage, and trusted key/connection
+attribution. Full mode includes the captured native input/output; usage mode omits
+content. First-byte timing supplies Langfuse's completion-start time as the gateway's
+first-token approximation. Relay outcome, provider status and capture completeness
+remain separate metadata fields. The exporter projects native OpenAI usage into the
+receiver's supported shape for pricing and preserves the untouched usage object as
+`native_usage` metadata. Missing usage is not reported as zero.
+
+Provisional upload limits are 32 concurrent tasks, 4 MiB serialized facts per record,
+16 MiB total retained serialized-fact/credential bytes, 8 MiB encoded payloads, and
+64 KiB ingestion responses. Serialization and mapping have additional bounded memory
+overhead; these byte budgets are not an RSS limit. Uploads have a two-second connect
+timeout and a five-second total timeout, shortened to the grant's remaining lifetime.
+Capacity exhaustion, oversized payloads, auth/HTTP errors, rejected OTLP spans, and
+transport failures are reported without changing inference results. Sanitized logs
+record failures/drops; shutdown reports accepted, failed and dropped counts.
+
+SIGTERM marks the gateway unready, drains inference, then finishes uploads within
+the remaining shared shutdown budget. At the deadline, unfinished uploads are
+cancelled and counted as drops. Process crashes or forced shutdown can lose telemetry;
+these records are not a durable accounting ledger.
+
+`telemetry/mod.rs` owns admission and task lifecycle, `telemetry/mapping.rs` maps facts,
+and `telemetry/otlp.rs` owns encoding and HTTP delivery. The uploader already accepts
+a span collection; future project batching can replace immediate scheduling while
+retaining each execution's original attribution/content policy and selecting a valid
+compatible grant. Capture and provider byte forwarding do not need to change.
+
## Response capture
Set `LANGFUSE_LOG_LEVEL=debug` to print one `gateway response captured` event at
@@ -190,8 +235,8 @@ output array at EOF. Missing usage remains null, rather than invented zeros.
`outcome` describes the relay (`eof`, `cancelled`, `timeout`, `transport_error`);
`provider_status` is separate. For example, a completed provider response can
still end with downstream cancellation. `first_byte_ms` measures the first body
-bytes observed by the gateway, not first-token latency. No completion-start time
-is fabricated from completed items.
+bytes observed by the gateway. Telemetry uses this as the first-token approximation
+for Langfuse completion-start time; it does not wait for completed output items.
`input_complete` means the full native request was captured; it is false when
content is omitted in usage mode. `output_complete` means response inspection
@@ -205,10 +250,10 @@ fully captured response, and downstream cancellation does not erase completed ca
The implementation separates shared `ExecutionCapture` lifecycle/timing, the
`OpenAiResponsesCapture` adapter and bounded `SseDecoder`. A private `ProtocolCapture`
enum dispatches to the adapter. Finalization hands an owned `InferenceFacts` record
-to `telemetry::record`, which currently only pretty-prints at debug level. Another
-provider can supply native facts through the same interface without changing the
-relay or final sink. No Anthropic adapter or Langfuse observation/span mapping is
-implemented yet. Capture runs independently of log level; only emission is gated.
+to telemetry for debug logging and immediate upload. Another provider can supply
+native facts through the same interface without changing the relay. The current
+usage mapping targets OpenAI Responses; no Anthropic adapter is implemented yet.
+Capture and upload run independently of log level; debug emission alone is gated.
Capture is limited to 1 MiB request inspection, 1 MiB JSON/SSE event inspection,
1 MiB retained output and 256 output items per execution. The active-request limit
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rs
index 2ceda4752e..03683f4151 100644
--- a/ai-gateway/src/capture/mod.rs
+++ b/ai-gateway/src/capture/mod.rs
@@ -13,7 +13,7 @@ use crate::{
resolution::{IngestionMode, MetadataValue, ResolvedRequestContext},
telemetry,
};
-use facts::ProviderFacts;
+pub(crate) use facts::ProviderFacts;
pub(crate) use facts::{InferenceFacts, RelayOutcome};
use openai_responses::OpenAiResponsesCapture;
@@ -61,6 +61,7 @@ pub(crate) struct ExecutionCapture {
first_byte_ms: Option<u128>,
http_status: Option<u16>,
metadata: Value,
+ delivery: Option<(telemetry::Telemetry, telemetry::DeliveryContext)>,
}
impl ExecutionCapture {
@@ -106,9 +107,21 @@ impl ExecutionCapture {
"key_metadata": key_metadata,
"ingestion_mode": if full { "full" } else { "usage" },
}),
+ delivery: None,
}
}
+ pub fn deliver_to(
+ &mut self,
+ telemetry: telemetry::Telemetry,
+ context: &ResolvedRequestContext,
+ ) {
+ self.delivery = Some((
+ telemetry,
+ telemetry::DeliveryContext::from_resolved(context),
+ ));
+ }
+
pub fn response(&mut self, status: u16, headers: &HeaderMap) {
if let Some(protocol) = &mut self.protocol {
self.http_status = Some(status);
@@ -137,7 +150,7 @@ impl ExecutionCapture {
return;
};
let (api_format, inference) = protocol.into_facts();
- telemetry::record(InferenceFacts {
+ let facts = InferenceFacts {
api_format,
start_time_unix_ms: self.start_time_unix_ms,
duration_ms: self.started.elapsed().as_millis(),
@@ -146,7 +159,11 @@ impl ExecutionCapture {
metadata: std::mem::take(&mut self.metadata),
outcome,
inference,
- });
+ };
+ telemetry::debug_record(&facts);
+ if let Some((telemetry, context)) = self.delivery.take() {
+ telemetry.record(context, facts);
+ }
}
}
diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rs
index fce29bf6bc..37fba9035c 100644
--- a/ai-gateway/src/inference.rs
+++ b/ai-gateway/src/inference.rs
@@ -16,6 +16,7 @@ pub struct InferenceService {
control_plane: ControlPlaneClient,
provider: OpenAiProvider,
resolution_capacity: Semaphore,
+ telemetry: Option<crate::telemetry::Telemetry>,
}
pub(crate) enum RequestPreparationError {
@@ -36,14 +37,21 @@ impl InferenceService {
if !(1..=Semaphore::MAX_PERMITS).contains(&max_concurrent_resolutions) {
return Err(ResolutionError::Configuration);
}
+ let telemetry = crate::telemetry::Telemetry::new(&config)?;
Ok(Self {
control_plane: ControlPlaneClient::new(config)?,
provider: OpenAiProvider::new(max_active_requests)
- .map_err(|_| ResolutionError::Configuration)?,
+ .map_err(|_| ResolutionError::Configuration)?
+ .with_telemetry(telemetry.clone()),
resolution_capacity: Semaphore::new(max_concurrent_resolutions),
+ telemetry: Some(telemetry),
})
}
+ pub fn telemetry(&self) -> Option<crate::telemetry::Telemetry> {
+ self.telemetry.clone()
+ }
+
/// Authenticate with a separate bounded budget before reserving execution capacity.
pub(crate) async fn resolve_and_admit(
&self,
@@ -86,6 +94,7 @@ impl InferenceService {
control_plane,
provider,
resolution_capacity: Semaphore::new(resolutions),
+ telemetry: None,
}
}
}
diff --git a/ai-gateway/src/lib.rs b/ai-gateway/src/lib.rs
index ce4f00cec5..1d06512e43 100644
--- a/ai-gateway/src/lib.rs
+++ b/ai-gateway/src/lib.rs
@@ -5,7 +5,7 @@ pub mod inference;
pub mod providers;
pub mod resolution;
pub mod server;
-mod telemetry;
+pub mod telemetry;
mod transport;
#[cfg(test)]
diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rs
index 3d4873dabd..94c87f689f 100644
--- a/ai-gateway/src/main.rs
+++ b/ai-gateway/src/main.rs
@@ -39,6 +39,7 @@ async fn run() -> Result<(), Box<dyn Error>> {
})
.transpose()?;
let inference_enabled = inference.is_some();
+ let telemetry = inference.as_ref().and_then(InferenceService::telemetry);
let state = if inference_enabled {
GatewayLifecycleState::default()
} else {
@@ -48,7 +49,24 @@ async fn run() -> Result<(), Box<dyn Error>> {
let shutdown = shutdown_signal()?;
let listener = bind_listener(config.listen_address, config.auto_increment_listen_port).await?;
tracing::info!(address = %listener.local_addr()?, inference_enabled, "gateway listening");
- server::serve(listener, app, state, shutdown, config.shutdown_timeout).await?;
+ let result = server::serve(
+ listener,
+ app,
+ state.clone(),
+ shutdown,
+ config.shutdown_timeout,
+ )
+ .await;
+ if let Some(telemetry) = telemetry {
+ telemetry
+ .shutdown(
+ state
+ .drain_deadline()
+ .unwrap_or_else(tokio::time::Instant::now),
+ )
+ .await;
+ }
+ result?;
tracing::info!("gateway stopped");
Ok(())
}
diff --git a/ai-gateway/src/providers/openai.rs b/ai-gateway/src/providers/openai.rs
index 71a6568056..7b05775e35 100644
--- a/ai-gateway/src/providers/openai.rs
+++ b/ai-gateway/src/providers/openai.rs
@@ -48,6 +48,7 @@ pub struct OpenAiProvider {
client: Client,
capacity: Arc<Semaphore>,
limits: ProviderLimits,
+ telemetry: Option<crate::telemetry::Telemetry>,
#[cfg(test)]
endpoint: String,
}
@@ -85,11 +86,17 @@ impl OpenAiProvider {
client,
capacity: Arc::new(Semaphore::new(limits.active)),
limits,
+ telemetry: None,
#[cfg(test)]
endpoint: "https://api.openai.com/v1/responses".to_owned(),
})
}
+ pub(crate) fn with_telemetry(mut self, telemetry: crate::telemetry::Telemetry) -> Self {
+ self.telemetry = Some(telemetry);
+ self
+ }
+
/// Reserve capacity for an authenticated request before reading its body.
///
/// # Errors
@@ -129,6 +136,9 @@ impl OpenAiProvider {
.map_err(|_| ProviderError::Configuration)?;
authorization.set_sensitive(true);
let mut capture = ExecutionCapture::openai_responses(&context, headers, &body);
+ if let Some(telemetry) = &self.telemetry {
+ capture.deliver_to(telemetry.clone(), &context);
+ }
let response = tokio::time::timeout_at(
permit
.deadline
diff --git a/ai-gateway/src/providers/openai/tests.rs b/ai-gateway/src/providers/openai/tests.rs
index 8c1d502e72..7cbb25a600 100644
--- a/ai-gateway/src/providers/openai/tests.rs
+++ b/ai-gateway/src/providers/openai/tests.rs
@@ -331,3 +331,209 @@ async fn deadlines_bound_headers_and_stalled_bodies_without_exposing_transport_d
assert!(!error.to_string().contains("provider-secret"));
assert!(relay.try_admit().is_ok());
}
+
+#[tokio::test]
+async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {
+ use crate::{
+ resolution::ControlPlaneConfig, telemetry::Telemetry,
+ test_support::resolved_request_context_with_mode,
+ };
+ use serde_json::Value;
+
+ for (content_type, native, sink_status) in [
+ (
+ "application/json",
+ r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#,
+ 200,
+ ),
+ (
+ "text/event-stream",
+ "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-1\",\"model\":\"actual\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":2,\"output_tokens\":1,\"total_tokens\":3}}}\n\n",
+ 503,
+ ),
+ ] {
+ let (sent, mut received) = tokio::sync::mpsc::channel(2);
+ let release = Arc::new(Notify::new());
+ let sink_release = release.clone();
+ let sink = FakeServer::start(move |request| {
+ let sent = sent.clone();
+ let release = sink_release.clone();
+ async move {
+ let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
+ sent.send(serde_json::from_slice::<Value>(&bytes).unwrap())
+ .await
+ .unwrap();
+ release.notified().await;
+ Response::builder()
+ .status(sink_status)
+ .body(Body::from("{}"))
+ .unwrap()
+ }
+ })
+ .await;
+ let telemetry =
+ Telemetry::new(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap()).unwrap();
+ let upstream = FakeServer::start(move |_| async move {
+ Response::builder()
+ .header("content-type", content_type)
+ .body(Body::from(native))
+ .unwrap()
+ })
+ .await;
+ let relay = provider(&upstream, 1).with_telemetry(telemetry.clone());
+ let context = resolved_request_context_with_mode("provider-secret", "full").await;
+ let response = relay
+ .forward(
+ relay.try_admit().unwrap(),
+ context,
+ &HeaderMap::new(),
+ Bytes::from_static(br#"{"model":"requested","input":"hello"}"#),
+ )
+ .await
+ .unwrap();
+ // Neither the last byte nor the provider permit waits for the sink response.
+ assert_eq!(
+ tokio::time::timeout(Duration::from_secs(1), to_bytes(response.into_body(), 4096))
+ .await
+ .unwrap()
+ .unwrap(),
+ native
+ );
+ assert!(relay.try_admit().is_ok());
+ let payload = tokio::time::timeout(Duration::from_secs(1), received.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ assert_completed_upload(&payload);
+ release.notify_one();
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(1))
+ .await;
+ assert!(received.try_recv().is_err());
+ assert_eq!(sink.calls(), 1);
+ assert_eq!(upstream.calls(), 1);
+ }
+}
+
+#[tokio::test]
+async fn cancelled_and_timed_out_executions_upload_after_provider_context_is_released() {
+ use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};
+ use serde_json::Value;
+
+ for cancelled in [true, false] {
+ let (sent, mut received) = tokio::sync::mpsc::channel(2);
+ let sink = FakeServer::start(move |request| {
+ let sent = sent.clone();
+ async move {
+ assert_eq!(
+ request.headers()[header::AUTHORIZATION],
+ "Bearer private-ingestion-token"
+ );
+ let bytes = to_bytes(request.into_body(), 65536).await.unwrap();
+ sent.send(serde_json::from_slice::<Value>(&bytes).unwrap())
+ .await
+ .unwrap();
+ Response::new(Body::from("{}"))
+ }
+ })
+ .await;
+ let telemetry =
+ Telemetry::new(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap()).unwrap();
+ let dropped = Arc::new(Notify::new());
+ let upstream = stalled_provider(dropped).await;
+ let relay = OpenAiProvider::for_test(
+ upstream.url.clone(),
+ ProviderLimits {
+ active: 1,
+ execution_timeout: Duration::from_millis(150),
+ ..ProviderLimits::default()
+ },
+ )
+ .with_telemetry(telemetry.clone());
+ let context = resolved_request_context("provider-secret").await;
+ let response = relay
+ .forward(
+ relay.try_admit().unwrap(),
+ context,
+ &HeaderMap::new(),
+ Bytes::from_static(br#"{"model":"requested","input":"prompt-canary"}"#),
+ )
+ .await
+ .unwrap();
+ let mut body = response.into_body().into_data_stream();
+ assert_eq!(body.next().await.unwrap().unwrap(), "first");
+ assert_eq!(sink.calls(), 0);
+ if cancelled {
+ drop(body);
+ } else {
+ assert!(body.next().await.unwrap().is_err());
+ drop(body);
+ }
+ let payload = tokio::time::timeout(Duration::from_secs(1), received.recv())
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(relay.try_admit().is_ok());
+ assert!(!payload.to_string().contains("prompt-canary"));
+ let attrs = payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]
+ .as_array()
+ .unwrap();
+ let metadata: Value = serde_json::from_str(
+ attrs
+ .iter()
+ .find(|a| a["key"] == "langfuse.observation.metadata")
+ .unwrap()["value"]["stringValue"]
+ .as_str()
+ .unwrap(),
+ )
+ .unwrap();
+ assert_eq!(
+ metadata["relay_outcome"],
+ if cancelled { "cancelled" } else { "timeout" }
+ );
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(1))
+ .await;
+ assert!(received.try_recv().is_err());
+ assert_eq!(sink.calls(), 1);
+ }
+}
+
+fn assert_completed_upload(payload: &serde_json::Value) {
+ use serde_json::{Value, json};
+ let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];
+ let attrs = span["attributes"].as_array().unwrap();
+ assert!(
+ attrs
+ .iter()
+ .any(|a| a["key"] == "langfuse.observation.model.name"
+ && a["value"]["stringValue"] == "actual")
+ );
+ assert!(
+ attrs
+ .iter()
+ .any(|a| a["key"] == "langfuse.observation.completion_start_time")
+ );
+ let metadata: Value = serde_json::from_str(
+ attrs
+ .iter()
+ .find(|a| a["key"] == "langfuse.observation.metadata")
+ .unwrap()["value"]["stringValue"]
+ .as_str()
+ .unwrap(),
+ )
+ .unwrap();
+ assert_eq!(metadata["relay_outcome"], "eof");
+ assert_eq!(
+ metadata["native_usage"],
+ json!({"input_tokens":2,"output_tokens":1,"total_tokens":3})
+ );
+ for secret in [
+ "provider-secret",
+ "private-ingestion-token",
+ "gateway-secret",
+ "service-key",
+ ] {
+ assert!(!payload.to_string().contains(secret));
+ }
+}
diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rs
index 83d1c61f19..fe5adfc95a 100644
--- a/ai-gateway/src/resolution/mod.rs
+++ b/ai-gateway/src/resolution/mod.rs
@@ -1,6 +1,6 @@
//! Resolve credentials through trusted Web before any provider execution.
mod contracts;
-mod signing;
+pub(crate) mod signing;
pub use contracts::{
ApiFormat, IngestionGrant, IngestionMode, MetadataValue, ProviderConnection,
@@ -25,7 +25,7 @@ struct ResolutionLimits {
/// Operator-supplied Web base URL and the existing gateway/Web service key.
pub struct ControlPlaneConfig {
- url: Url,
+ web_url: Url,
service_key: String,
limits: ResolutionLimits,
}
@@ -38,7 +38,7 @@ impl ControlPlaneConfig {
/// base URL. URLs require HTTPS (HTTP is allowed on loopback), with no userinfo,
/// query, or fragment.
pub fn new(web_url: &str, service_key: &str) -> Result<Self, ResolutionError> {
- let mut url = Url::parse(web_url).map_err(|_| ResolutionError::Configuration)?;
+ let url = Url::parse(web_url).map_err(|_| ResolutionError::Configuration)?;
let loopback = url.host_str().is_some_and(|host| {
host == "localhost"
|| host
@@ -56,10 +56,8 @@ impl ControlPlaneConfig {
{
return Err(ResolutionError::Configuration);
}
- let resolve_path = format!("{}{RESOLVE_PATH}", url.path().trim_end_matches('/'));
- url.set_path(&resolve_path);
Ok(Self {
- url,
+ web_url: url,
service_key: service_key.to_owned(),
limits: ResolutionLimits {
timeout: Duration::from_secs(5),
@@ -67,6 +65,16 @@ impl ControlPlaneConfig {
},
})
}
+
+ pub(crate) fn endpoint(&self, path: &str) -> Url {
+ let mut url = self.web_url.clone();
+ url.set_path(&format!("{}{path}", url.path().trim_end_matches('/')));
+ url
+ }
+
+ pub(crate) fn service_key(&self) -> &str {
+ &self.service_key
+ }
}
/// Reuses the HTTP connection pool; credentials belong exclusively to each request.
@@ -154,7 +162,7 @@ impl ControlPlaneClient {
.map_err(|_| ResolutionError::Configuration)?;
let mut response = self
.client
- .post(self.config.url.clone())
+ .post(self.config.endpoint(RESOLVE_PATH))
.header(AUTHORIZATION, credential)
.header("langfuse-gateway-authorization", signature)
.header(CONTENT_TYPE, "application/json")
diff --git a/ai-gateway/src/resolution/signing.rs b/ai-gateway/src/resolution/signing.rs
index 0e5303f4a8..776314d920 100644
--- a/ai-gateway/src/resolution/signing.rs
+++ b/ai-gateway/src/resolution/signing.rs
@@ -2,8 +2,8 @@ use hmac::{Hmac, KeyInit, Mac};
use sha2::{Digest, Sha256};
use std::fmt::Write;
-pub(super) fn authorization(service_key: &str, gateway_key: &str, timestamp: u64) -> String {
- let key_hash = lowercase_hex(&Sha256::digest(gateway_key.as_bytes()));
+pub(crate) fn authorization(service_key: &str, credential: &str, timestamp: u64) -> String {
+ let key_hash = lowercase_hex(&Sha256::digest(credential.as_bytes()));
let payload = format!("gateway-web-v1\n{timestamp}\n{key_hash}");
// HMAC accepts keys of any length.
let mut mac = Hmac::<Sha256>::new_from_slice(service_key.as_bytes()).expect("valid HMAC key");
diff --git a/ai-gateway/src/server.rs b/ai-gateway/src/server.rs
index e0a65e9d89..70e82dcdf8 100644
--- a/ai-gateway/src/server.rs
+++ b/ai-gateway/src/server.rs
@@ -2,7 +2,7 @@ use std::{
future::{Future, IntoFuture},
io,
sync::{
- Arc,
+ Arc, Mutex,
atomic::{AtomicBool, Ordering},
},
time::Duration,
@@ -16,9 +16,21 @@ use tokio::{net::TcpListener, sync::oneshot};
pub struct GatewayLifecycleState {
ready: Arc<AtomicBool>,
disabled: bool,
+ drain_deadline: Arc<Mutex<Option<tokio::time::Instant>>>,
}
impl GatewayLifecycleState {
+ /// Deadline shared by connection draining and telemetry delivery.
+ ///
+ /// # Panics
+ /// Panics if the lifecycle lock was poisoned.
+ pub fn drain_deadline(&self) -> Option<tokio::time::Instant> {
+ *self
+ .drain_deadline
+ .lock()
+ .expect("gateway drain lock poisoned")
+ }
+
pub fn is_ready(&self) -> bool {
!self.disabled && self.ready.load(Ordering::Acquire)
}
@@ -67,6 +79,9 @@ async fn readiness(State(state): State<GatewayLifecycleState>) -> (StatusCode, J
/// # Errors
/// Returns the server's I/O error, or [`io::ErrorKind::TimedOut`] if connections
/// do not finish draining within `shutdown_timeout`.
+///
+/// # Panics
+/// Panics if the lifecycle lock was poisoned.
pub async fn serve(
listener: TcpListener,
app: Router,
@@ -80,6 +95,11 @@ pub async fn serve(
.with_graceful_shutdown(async move {
shutdown.await;
shutdown_state.ready.store(false, Ordering::Release);
+ *shutdown_state
+ .drain_deadline
+ .lock()
+ .expect("gateway drain lock poisoned") =
+ Some(tokio::time::Instant::now() + shutdown_timeout);
tracing::info!("gateway draining");
let _ = draining_tx.send(());
})
diff --git a/ai-gateway/src/telemetry.rs b/ai-gateway/src/telemetry.rs
deleted file mode 100644
index bedc5daaac..0000000000
--- a/ai-gateway/src/telemetry.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-//! Final facts delivery. A Langfuse exporter can replace this debug sink later.
-use crate::capture::InferenceFacts;
-
-pub(crate) fn record(facts: InferenceFacts) {
- // Full mode intentionally includes native request and completed output content.
- tracing::debug!(capture = %format_args!("{:#}", serde_json::to_value(facts).expect("captured facts must serialize")), "gateway response captured");
-}
diff --git a/ai-gateway/src/telemetry/mapping.rs b/ai-gateway/src/telemetry/mapping.rs
new file mode 100644
index 0000000000..9b61012e3a
--- /dev/null
+++ b/ai-gateway/src/telemetry/mapping.rs
@@ -0,0 +1,310 @@
+//! Pure mapping from finalized execution facts to a Langfuse generation span.
+use chrono::{DateTime, SecondsFormat};
+use serde_json::{Map, Value, json};
+
+use crate::capture::{InferenceFacts, RelayOutcome};
+
+pub(super) fn span(facts: InferenceFacts, trace_id: &str, observation_id: &str) -> Value {
+ let full = facts.metadata.get("ingestion_mode").and_then(Value::as_str) == Some("full");
+ let (level, message) = observation_status(&facts);
+ let inference = facts.inference;
+ let mut metadata = facts.metadata.as_object().cloned().unwrap_or_default();
+ metadata.extend([
+ ("api_format".into(), json!(facts.api_format)),
+ ("relay_outcome".into(), json!(facts.outcome)),
+ ("http_status".into(), json!(facts.http_status)),
+ ("first_byte_ms".into(), json!(facts.first_byte_ms)),
+ ("requested_model".into(), json!(inference.requested_model)),
+ (
+ "provider_response_id".into(),
+ json!(inference.provider_response_id),
+ ),
+ (
+ "provider_request_id".into(),
+ json!(inference.provider_request_id),
+ ),
+ ("provider_status".into(), json!(inference.provider_status)),
+ ("input_complete".into(), json!(inference.input_complete)),
+ ("output_complete".into(), json!(inference.output_complete)),
+ ("capture_complete".into(), json!(inference.capture_complete)),
+ ]);
+ let mut attributes = vec![attribute("langfuse.observation.type", "generation")];
+ if let Some(model) = inference.model.or(inference.requested_model) {
+ attributes.push(attribute("langfuse.observation.model.name", model));
+ }
+ if !inference.model_parameters.is_empty() {
+ attributes.push(attribute(
+ "langfuse.observation.model.parameters",
+ Value::Object(inference.model_parameters).to_string(),
+ ));
+ }
+ if let Some(usage) = inference.usage_details {
+ if let Some(projected) = openai_usage(&usage) {
+ attributes.push(attribute(
+ "langfuse.observation.usage_details",
+ projected.to_string(),
+ ));
+ }
+ metadata.insert("native_usage".into(), usage);
+ }
+ if full {
+ if let Some(input) = inference.input {
+ attributes.push(attribute("langfuse.observation.input", input.to_string()));
+ }
+ if let Some(output) = inference.output {
+ attributes.push(attribute("langfuse.observation.output", output.to_string()));
+ }
+ }
+ if let Some(completion_start) = facts
+ .first_byte_ms
+ .and_then(|elapsed| facts.start_time_unix_ms.checked_add(elapsed))
+ .and_then(|timestamp| i64::try_from(timestamp).ok())
+ .and_then(DateTime::from_timestamp_millis)
+ {
+ attributes.push(attribute(
+ "langfuse.observation.completion_start_time",
+ completion_start.to_rfc3339_opts(SecondsFormat::Millis, true),
+ ));
+ }
+ attributes.push(attribute("langfuse.observation.level", level));
+ if let Some(message) = message {
+ attributes.push(attribute("langfuse.observation.status_message", message));
+ }
+ attributes.push(attribute(
+ "langfuse.observation.metadata",
+ Value::Object(metadata).to_string(),
+ ));
+ json!({
+ "traceId": trace_id,
+ "spanId": observation_id,
+ "name": facts.api_format,
+ "kind": 3,
+ "startTimeUnixNano": facts.start_time_unix_ms.saturating_mul(1_000_000).to_string(),
+ "endTimeUnixNano": facts.start_time_unix_ms.saturating_add(facts.duration_ms).saturating_mul(1_000_000).to_string(),
+ "attributes": attributes,
+ "status": { "code": if level == "ERROR" { 2 } else { 0 } },
+ })
+}
+
+fn observation_status(facts: &InferenceFacts) -> (&'static str, Option<&'static str>) {
+ let failure = if facts.http_status.is_some_and(|status| status >= 400) {
+ Some("Provider HTTP error")
+ } else if facts.inference.provider_status.as_deref() == Some("failed") {
+ Some("Provider response failed")
+ } else {
+ match facts.outcome {
+ RelayOutcome::Timeout => Some("Provider response timed out"),
+ RelayOutcome::TransportError => Some("Provider transport error"),
+ RelayOutcome::Eof | RelayOutcome::Cancelled => None,
+ }
+ };
+ if let Some(message) = failure {
+ ("ERROR", Some(message))
+ } else if facts.outcome == RelayOutcome::Cancelled {
+ ("WARNING", Some("Client cancelled the response"))
+ } else if facts.inference.provider_status.as_deref() == Some("incomplete") {
+ ("WARNING", Some("Provider response incomplete"))
+ } else {
+ ("DEFAULT", None)
+ }
+}
+
+fn attribute(key: &str, value: impl Into<String>) -> Value {
+ json!({"key": key, "value": {"stringValue": value.into()}})
+}
+
+/// The receiver's native `OpenAI` usage schema is strict at the top level, but
+/// accepts new numeric detail counters. Keep the original payload in metadata.
+fn openai_usage(usage: &Value) -> Option<Value> {
+ let usage = usage.as_object()?;
+ let mut projected = Map::new();
+ for key in ["input_tokens", "output_tokens", "total_tokens"] {
+ let value = usage.get(key).filter(|value| value.as_u64().is_some())?;
+ projected.insert(key.into(), value.clone());
+ }
+ for key in ["input_tokens_details", "output_tokens_details"] {
+ match usage.get(key) {
+ Some(Value::Null) => {
+ projected.insert(key.into(), Value::Null);
+ }
+ Some(Value::Object(details)) => {
+ projected.insert(
+ key.into(),
+ Value::Object(
+ details
+ .iter()
+ .filter(|(_, value)| value.is_null() || value.as_u64().is_some())
+ .map(|(key, value)| (key.clone(), value.clone()))
+ .collect(),
+ ),
+ );
+ }
+ _ => {}
+ }
+ }
+ Some(Value::Object(projected))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn facts() -> InferenceFacts {
+ let mut facts = InferenceFacts {
+ api_format: "openai.responses",
+ start_time_unix_ms: 1_735_689_600_000,
+ duration_ms: 900,
+ first_byte_ms: Some(125),
+ http_status: Some(200),
+ metadata: json!({"ingestion_mode": "full", "project_id": "project", "key_id": "key", "key_metadata": {"team": "search"}}),
+ outcome: RelayOutcome::Eof,
+ inference: crate::capture::ProviderFacts::default(),
+ };
+ facts.inference.model = Some("actual-model".into());
+ facts.inference.requested_model = Some("requested-model".into());
+ facts
+ .inference
+ .model_parameters
+ .insert("service_tier".into(), json!("priority"));
+ facts.inference.input = Some(json!({"input": "hello", "tools": [{"type": "function"}]}));
+ facts.inference.output = Some(json!([{ "type": "message", "content": "world" }]));
+ facts.inference.capture_complete = true;
+ facts.inference.output_complete = true;
+ facts.inference.input_complete = true;
+ facts
+ }
+
+ fn attributes(span: &Value) -> Map<String, Value> {
+ span["attributes"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|attribute| {
+ (
+ attribute["key"].as_str().unwrap().into(),
+ attribute["value"]["stringValue"].clone(),
+ )
+ })
+ .collect()
+ }
+
+ fn metadata(attributes: &Map<String, Value>) -> Value {
+ serde_json::from_str(
+ attributes["langfuse.observation.metadata"]
+ .as_str()
+ .unwrap(),
+ )
+ .unwrap()
+ }
+
+ #[test]
+ fn full_generation_preserves_native_content_attribution_and_first_byte_time() {
+ let span = span(
+ facts(),
+ "0123456789abcdef0123456789abcdef",
+ "0123456789abcdef",
+ );
+ assert_eq!(span["traceId"], "0123456789abcdef0123456789abcdef");
+ assert_eq!(span["spanId"], "0123456789abcdef");
+ assert!(span.get("parentSpanId").is_none());
+ assert_eq!(span["startTimeUnixNano"], "1735689600000000000");
+ assert_eq!(span["endTimeUnixNano"], "1735689600900000000");
+ let attrs = attributes(&span);
+ assert_eq!(attrs["langfuse.observation.type"], "generation");
+ assert_eq!(attrs["langfuse.observation.model.name"], "actual-model");
+ assert_eq!(
+ attrs["langfuse.observation.completion_start_time"],
+ "2025-01-01T00:00:00.125Z"
+ );
+ assert_eq!(
+ serde_json::from_str::<Value>(attrs["langfuse.observation.input"].as_str().unwrap())
+ .unwrap()["tools"][0]["type"],
+ "function"
+ );
+ assert!(attrs.contains_key("langfuse.observation.output"));
+ let metadata = metadata(&attrs);
+ assert_eq!(metadata["project_id"], "project");
+ assert_eq!(metadata["key_metadata"]["team"], "search");
+ assert_eq!(metadata["requested_model"], "requested-model");
+ assert_eq!(metadata["capture_complete"], true);
+ }
+
+ #[test]
+ fn usage_mode_omits_content_and_missing_usage_and_uses_requested_model() {
+ let mut facts = facts();
+ facts.metadata["ingestion_mode"] = json!("usage");
+ facts.inference.model = None;
+ facts.first_byte_ms = None;
+ let attrs = attributes(&span(facts, "trace", "span"));
+ for key in ["input", "output", "usage_details", "completion_start_time"] {
+ assert!(!attrs.contains_key(&format!("langfuse.observation.{key}")));
+ }
+ assert_eq!(attrs["langfuse.observation.model.name"], "requested-model");
+ assert!(metadata(&attrs).get("native_usage").is_none());
+ }
+
+ #[test]
+ fn usage_projection_preserves_new_numeric_details_without_unknown_top_level_fields() {
+ let mut facts = facts();
+ let usage = json!({
+ "input_tokens": 20, "output_tokens": 10, "total_tokens": 30,
+ "input_tokens_details": {"cached_tokens": 4, "future_counter": 3, "optional": null, "future_object": {}},
+ "output_tokens_details": null,
+ "future_usage": {"cost": 123},
+ });
+ facts.inference.usage_details = Some(usage.clone());
+ let attrs = attributes(&span(facts, "trace", "span"));
+ assert_eq!(metadata(&attrs)["native_usage"], usage);
+ let projected: Value = serde_json::from_str(
+ attrs["langfuse.observation.usage_details"]
+ .as_str()
+ .unwrap(),
+ )
+ .unwrap();
+ assert_eq!(
+ projected,
+ json!({
+ "input_tokens": 20, "output_tokens": 10, "total_tokens": 30,
+ "input_tokens_details": {"cached_tokens": 4, "future_counter": 3, "optional": null},
+ "output_tokens_details": null,
+ })
+ );
+ }
+
+ #[test]
+ fn unsupported_usage_remains_metadata_without_invented_totals() {
+ let mut facts = facts();
+ facts.inference.usage_details = Some(json!({"input_tokens": 9}));
+ let attrs = attributes(&span(facts, "trace", "span"));
+ assert!(!attrs.contains_key("langfuse.observation.usage_details"));
+ assert_eq!(metadata(&attrs)["native_usage"], json!({"input_tokens": 9}));
+ }
+
+ #[test]
+ fn provider_and_transport_failures_are_errors_while_cancellation_stays_distinct() {
+ for (http_status, provider_status, outcome, level, code) in [
+ (Some(429), None, RelayOutcome::Eof, "ERROR", 2),
+ (Some(200), Some("failed"), RelayOutcome::Eof, "ERROR", 2),
+ (None, None, RelayOutcome::TransportError, "ERROR", 2),
+ (Some(200), None, RelayOutcome::Timeout, "ERROR", 2),
+ (Some(200), None, RelayOutcome::Cancelled, "WARNING", 0),
+ (
+ Some(200),
+ Some("incomplete"),
+ RelayOutcome::Eof,
+ "WARNING",
+ 0,
+ ),
+ ] {
+ let mut facts = facts();
+ facts.http_status = http_status;
+ facts.inference.provider_status = provider_status.map(str::to_owned);
+ facts.outcome = outcome;
+ let span = span(facts, "trace", "span");
+ let attrs = attributes(&span);
+ assert_eq!(attrs["langfuse.observation.level"], level);
+ assert_eq!(span["status"]["code"], code);
+ assert_eq!(metadata(&attrs)["relay_outcome"], json!(outcome));
+ }
+ }
+}
diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rs
new file mode 100644
index 0000000000..7a9b2383a3
--- /dev/null
+++ b/ai-gateway/src/telemetry/mod.rs
@@ -0,0 +1,208 @@
+//! Immediate, best-effort delivery of finalized inference facts.
+mod mapping;
+mod otlp;
+
+use std::{
+ io::{self, Write},
+ sync::{
+ Arc, Mutex,
+ atomic::{AtomicU64, Ordering},
+ },
+};
+
+use tokio::{sync::Semaphore, task::JoinSet, time::Instant};
+use uuid::Uuid;
+
+use crate::{
+ capture::InferenceFacts,
+ resolution::{ControlPlaneConfig, ResolutionError, ResolvedRequestContext},
+};
+use otlp::Uploader;
+
+const MAX_UPLOADS: usize = 32;
+const MAX_FACT_BYTES: usize = 4 * 1024 * 1024;
+const MAX_RETAINED_BYTES: usize = 16 * 1024 * 1024;
+
+/// Credentials are deliberately not serializable or printable. Project attribution
+/// stays attached to each execution even when delivery outlives the response body.
+pub(crate) struct DeliveryContext {
+ pub project_id: String,
+ pub access_token: String,
+ pub expires_at: u64,
+ trace_id: String,
+ observation_id: String,
+}
+
+impl DeliveryContext {
+ pub fn from_resolved(context: &ResolvedRequestContext) -> Self {
+ Self {
+ project_id: context.attribution().project_id().to_owned(),
+ access_token: context.ingestion().access_token().to_owned(),
+ expires_at: context.ingestion().expires_at(),
+ trace_id: Uuid::new_v4().simple().to_string(),
+ observation_id: Uuid::new_v4().simple().to_string()[..16].to_owned(),
+ }
+ }
+}
+
+#[derive(Default)]
+struct Stats {
+ accepted: AtomicU64,
+ failed: AtomicU64,
+ dropped: AtomicU64,
+}
+
+struct Delivery {
+ uploader: Arc<Uploader>,
+ capacity: Arc<Semaphore>,
+ bytes: Arc<Semaphore>,
+ tasks: Mutex<Option<JoinSet<()>>>,
+ stats: Arc<Stats>,
+}
+
+/// Shared delivery handle. Shutdown closes submission and drains already admitted work.
+#[derive(Clone)]
+pub struct Telemetry(Arc<Delivery>);
+
+impl Telemetry {
+ pub(crate) fn new(config: &ControlPlaneConfig) -> Result<Self, ResolutionError> {
+ Ok(Self::with_uploader(
+ Uploader::new(config)?,
+ MAX_UPLOADS,
+ MAX_RETAINED_BYTES,
+ ))
+ }
+
+ fn with_uploader(uploader: Uploader, uploads: usize, bytes: usize) -> Self {
+ Self(Arc::new(Delivery {
+ uploader: Arc::new(uploader),
+ capacity: Arc::new(Semaphore::new(uploads)),
+ bytes: Arc::new(Semaphore::new(bytes)),
+ tasks: Mutex::new(Some(JoinSet::new())),
+ stats: Arc::new(Stats::default()),
+ }))
+ }
+
+ /// No waiting queue and no network work on the caller's response/drop path.
+ pub(crate) fn record(&self, context: DeliveryContext, facts: InferenceFacts) {
+ let Ok(capacity) = self.0.capacity.clone().try_acquire_owned() else {
+ self.drop_record("capacity");
+ return;
+ };
+ let mut size = SizeCounter {
+ bytes: context.access_token.len() + context.project_id.len(),
+ limit: MAX_FACT_BYTES,
+ };
+ if serde_json::to_writer(&mut size, &facts).is_err() {
+ self.drop_record("size");
+ return;
+ }
+ let Ok(bytes) = self
+ .0
+ .bytes
+ .clone()
+ .try_acquire_many_owned(u32::try_from(size.bytes).expect("bounded fact bytes"))
+ else {
+ self.drop_record("bytes");
+ return;
+ };
+ let mut tasks = self.0.tasks.lock().expect("telemetry task lock poisoned");
+ let Some(tasks) = tasks.as_mut() else {
+ self.drop_record("shutdown");
+ return;
+ };
+ // Reap completed handles on submission so the task registry stays bounded.
+ while tasks.try_join_next().is_some() {}
+ let uploader = self.0.uploader.clone();
+ let stats = self.0.stats.clone();
+ tasks.spawn(async move {
+ let _capacity = capacity;
+ let _bytes = bytes;
+ let span = mapping::span(facts, &context.trace_id, &context.observation_id);
+ match uploader.export(&context, &[span]).await {
+ Ok(()) => {
+ stats.accepted.fetch_add(1, Ordering::Relaxed);
+ tracing::debug!("gateway telemetry accepted");
+ }
+ Err(error) => {
+ stats.failed.fetch_add(1, Ordering::Relaxed);
+ // Error categories contain no URLs, credentials, response bodies or content.
+ tracing::warn!(reason = error.reason(), "gateway telemetry upload failed");
+ }
+ }
+ });
+ }
+
+ fn drop_record(&self, reason: &'static str) {
+ self.0.stats.dropped.fetch_add(1, Ordering::Relaxed);
+ tracing::warn!(reason, "gateway telemetry dropped");
+ }
+
+ /// Stop accepting work and finish uploads within the existing process drain deadline.
+ ///
+ /// # Panics
+ /// Panics if the upload task lock was poisoned.
+ pub async fn shutdown(&self, deadline: Instant) {
+ self.0.capacity.close();
+ let tasks = self
+ .0
+ .tasks
+ .lock()
+ .expect("telemetry task lock poisoned")
+ .take();
+ let Some(mut tasks) = tasks else {
+ return;
+ };
+ loop {
+ match tokio::time::timeout_at(deadline, tasks.join_next()).await {
+ Ok(Some(Ok(()))) => {}
+ Ok(Some(Err(_))) => {
+ self.0.stats.failed.fetch_add(1, Ordering::Relaxed);
+ }
+ Ok(None) => break,
+ Err(_) => {
+ tasks.abort_all();
+ while let Some(result) = tasks.join_next().await {
+ if result.is_err() {
+ self.0.stats.dropped.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ break;
+ }
+ }
+ }
+ tracing::info!(
+ accepted = self.0.stats.accepted.load(Ordering::Relaxed),
+ failed = self.0.stats.failed.load(Ordering::Relaxed),
+ dropped = self.0.stats.dropped.load(Ordering::Relaxed),
+ "gateway telemetry stopped"
+ );
+ }
+}
+
+/// Count serialized facts without allocating a second copy on the finalization path.
+struct SizeCounter {
+ bytes: usize,
+ limit: usize,
+}
+
+impl Write for SizeCounter {
+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+ if bytes.len() > self.limit.saturating_sub(self.bytes) {
+ return Err(io::Error::other("telemetry size limit"));
+ }
+ self.bytes += bytes.len();
+ Ok(bytes.len())
+ }
+ fn flush(&mut self) -> io::Result<()> {
+ Ok(())
+ }
+}
+
+pub(crate) fn debug_record(facts: &InferenceFacts) {
+ // Full mode intentionally includes native request and completed output content.
+ tracing::debug!(capture = %format_args!("{:#}", serde_json::to_value(facts).expect("captured facts must serialize")), "gateway response captured");
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/ai-gateway/src/telemetry/otlp.rs b/ai-gateway/src/telemetry/otlp.rs
new file mode 100644
index 0000000000..c35a5d1528
--- /dev/null
+++ b/ai-gateway/src/telemetry/otlp.rs
@@ -0,0 +1,165 @@
+use std::{
+ io::{self, Write},
+ time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use axum::http::{HeaderValue, header};
+use reqwest::{Client, Url};
+use serde_json::{Value, json};
+
+use super::DeliveryContext;
+use crate::resolution::{ControlPlaneConfig, ResolutionError, signing};
+
+const INGESTION_PATH: &str = "/api/public/otel/v1/traces";
+const MAX_PAYLOAD_BYTES: usize = 8 * 1024 * 1024;
+const MAX_RESPONSE_BYTES: usize = 64 * 1024;
+const UPLOAD_TIMEOUT: Duration = Duration::from_secs(5);
+
+pub(super) struct Uploader {
+ client: Client,
+ endpoint: Url,
+ service_key: String,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub(super) enum ExportError {
+ Expired,
+ Payload,
+ Transport,
+ Rejected,
+ Response,
+ Partial,
+}
+
+impl ExportError {
+ pub fn reason(&self) -> &'static str {
+ match self {
+ Self::Expired => "expired_grant",
+ Self::Payload => "payload",
+ Self::Transport => "transport",
+ Self::Rejected => "http_status",
+ Self::Response => "invalid_response",
+ Self::Partial => "rejected_spans",
+ }
+ }
+}
+
+impl Uploader {
+ pub fn new(config: &ControlPlaneConfig) -> Result<Self, ResolutionError> {
+ let client = Client::builder()
+ .redirect(reqwest::redirect::Policy::none())
+ .retry(reqwest::retry::never())
+ .no_proxy()
+ .no_gzip()
+ .no_brotli()
+ .no_deflate()
+ .no_zstd()
+ .connect_timeout(Duration::from_secs(2))
+ .timeout(UPLOAD_TIMEOUT)
+ .pool_max_idle_per_host(16)
+ .build()
+ .map_err(|_| ResolutionError::Configuration)?;
+ Ok(Self {
+ client,
+ endpoint: config.endpoint(INGESTION_PATH),
+ service_key: config.service_key().to_owned(),
+ })
+ }
+
+ /// A collection allows project batching without changing the HTTP transport.
+ pub async fn export(
+ &self,
+ grant: &DeliveryContext,
+ spans: &[Value],
+ ) -> Result<(), ExportError> {
+ let payload = json!({"resourceSpans": [{
+ "resource": {"attributes": [{"key": "service.name", "value": {"stringValue": "langfuse-ai-gateway"}}]},
+ "scopeSpans": [{"scope": {"name": "langfuse-ai-gateway", "version": env!("CARGO_PKG_VERSION")}, "spans": spans}]
+ }]});
+ let mut body = LimitedBody(Vec::new());
+ serde_json::to_writer(&mut body, &payload).map_err(|_| ExportError::Payload)?;
+ let now = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map_err(|_| ExportError::Expired)?;
+ let remaining = Duration::from_secs(grant.expires_at)
+ .checked_sub(now)
+ .filter(|duration| !duration.is_zero())
+ .ok_or(ExportError::Expired)?;
+ let mut authorization = HeaderValue::from_str(&format!("Bearer {}", grant.access_token))
+ .map_err(|_| ExportError::Payload)?;
+ authorization.set_sensitive(true);
+ let mut signature = HeaderValue::from_str(&signing::authorization(
+ &self.service_key,
+ &grant.access_token,
+ now.as_secs(),
+ ))
+ .map_err(|_| ExportError::Payload)?;
+ signature.set_sensitive(true);
+ let mut response = self
+ .client
+ .post(self.endpoint.clone())
+ .header(header::AUTHORIZATION, authorization)
+ .header("langfuse-gateway-authorization", signature)
+ .header(header::CONTENT_TYPE, "application/json")
+ .header(header::ACCEPT_ENCODING, "identity")
+ .header("x-langfuse-sdk-name", "langfuse-ai-gateway")
+ .header("x-langfuse-sdk-version", env!("CARGO_PKG_VERSION"))
+ .header("x-langfuse-ingestion-version", "4")
+ .timeout(UPLOAD_TIMEOUT.min(remaining))
+ .body(body.0)
+ .send()
+ .await
+ .map_err(|_| ExportError::Transport)?;
+ if response.status().as_u16() != 200 {
+ return Err(ExportError::Rejected);
+ }
+ if response
+ .content_length()
+ .is_some_and(|size| size > MAX_RESPONSE_BYTES as u64)
+ {
+ return Err(ExportError::Response);
+ }
+ let mut bytes = Vec::new();
+ while let Some(chunk) = response.chunk().await.map_err(|_| ExportError::Transport)? {
+ if chunk.len() > MAX_RESPONSE_BYTES - bytes.len() {
+ return Err(ExportError::Response);
+ }
+ bytes.extend_from_slice(&chunk);
+ }
+ let result: Value = serde_json::from_slice(&bytes).map_err(|_| ExportError::Response)?;
+ if !result.is_object() {
+ return Err(ExportError::Response);
+ }
+ if let Some(partial) = result.get("partialSuccess") {
+ if !partial.is_object() {
+ return Err(ExportError::Response);
+ }
+ let rejected = match partial.get("rejectedSpans") {
+ None => 0,
+ Some(Value::String(value)) => {
+ value.parse::<u64>().map_err(|_| ExportError::Response)?
+ }
+ Some(value) => value.as_u64().ok_or(ExportError::Response)?,
+ };
+ if rejected > 0 {
+ return Err(ExportError::Partial);
+ }
+ }
+ Ok(())
+ }
+}
+
+struct LimitedBody(Vec<u8>);
+
+impl Write for LimitedBody {
+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+ if bytes.len() > MAX_PAYLOAD_BYTES - self.0.len() {
+ return Err(io::Error::other("telemetry payload limit"));
+ }
+ self.0.extend_from_slice(bytes);
+ Ok(bytes.len())
+ }
+ fn flush(&mut self) -> io::Result<()> {
+ Ok(())
+ }
+}
diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rs
new file mode 100644
index 0000000000..2b1bd50c3a
--- /dev/null
+++ b/ai-gateway/src/telemetry/tests.rs
@@ -0,0 +1,320 @@
+use std::{sync::Arc, time::Duration};
+
+use axum::{
+ body::{Body, to_bytes},
+ http::Response,
+};
+use serde_json::{Value, json};
+use tokio::sync::Notify;
+
+use super::{otlp::ExportError, *};
+use crate::{
+ capture::RelayOutcome,
+ resolution::signing,
+ test_support::{FakeServer, resolved_request_context},
+};
+
+fn facts(project: &str) -> InferenceFacts {
+ InferenceFacts {
+ api_format: "openai.responses",
+ start_time_unix_ms: 1_735_689_600_000,
+ duration_ms: 100,
+ first_byte_ms: Some(10),
+ http_status: Some(200),
+ metadata: json!({"project_id": project, "ingestion_mode": "usage"}),
+ outcome: RelayOutcome::Eof,
+ inference: crate::capture::ProviderFacts::default(),
+ }
+}
+
+async fn grant() -> DeliveryContext {
+ DeliveryContext::from_resolved(&resolved_request_context("provider-secret").await)
+}
+
+fn uploader(url: &str) -> Uploader {
+ Uploader::new(&ControlPlaneConfig::new(url, "test-service-key").unwrap()).unwrap()
+}
+
+fn response(status: u16, body: impl Into<Body>) -> Response<Body> {
+ Response::builder()
+ .status(status)
+ .body(body.into())
+ .unwrap()
+}
+
+#[tokio::test]
+async fn immediate_upload_uses_prefixed_path_signed_grant_and_gateway_sdk_headers() {
+ let web = FakeServer::start(|request| async move {
+ assert_eq!(request.method(), "POST");
+ assert_eq!(request.uri().path(), "/app/api/public/otel/v1/traces");
+ let headers = request.headers();
+ assert_eq!(headers["authorization"], "Bearer private-ingestion-token");
+ let signature = headers["langfuse-gateway-authorization"].to_str().unwrap();
+ let timestamp = signature
+ .strip_prefix("HMAC timestamp=")
+ .unwrap()
+ .split_once(',')
+ .unwrap()
+ .0
+ .parse()
+ .unwrap();
+ assert_eq!(
+ signature,
+ signing::authorization("test-service-key", "private-ingestion-token", timestamp)
+ );
+ assert_eq!(headers["content-type"], "application/json");
+ assert_eq!(headers["x-langfuse-sdk-name"], "langfuse-ai-gateway");
+ assert_eq!(headers["x-langfuse-sdk-version"], env!("CARGO_PKG_VERSION"));
+ assert_eq!(headers["x-langfuse-ingestion-version"], "4");
+ let bytes = to_bytes(request.into_body(), 64 * 1024).await.unwrap();
+ let payload: Value = serde_json::from_slice(&bytes).unwrap();
+ let spans = payload["resourceSpans"][0]["scopeSpans"][0]["spans"]
+ .as_array()
+ .unwrap();
+ assert_eq!(spans.len(), 1);
+ assert_eq!(spans[0]["traceId"].as_str().unwrap().len(), 32);
+ assert_eq!(spans[0]["spanId"].as_str().unwrap().len(), 16);
+ let body = String::from_utf8(bytes.to_vec()).unwrap();
+ assert!(!body.contains("provider-secret"));
+ assert!(!body.contains("private-ingestion-token"));
+ response(200, "{}")
+ })
+ .await;
+ let telemetry = Telemetry::new(
+ &ControlPlaneConfig::new(&format!("{}/app", web.url), "test-service-key").unwrap(),
+ )
+ .unwrap();
+ telemetry.record(grant().await, facts("project-1"));
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(2))
+ .await;
+ assert_eq!(web.calls(), 1);
+ assert_eq!(telemetry.0.stats.accepted.load(Ordering::Relaxed), 1);
+ assert_eq!(telemetry.0.stats.failed.load(Ordering::Relaxed), 0);
+}
+
+#[tokio::test]
+async fn concurrent_projects_keep_their_original_grants_and_attribution() {
+ let observed = Arc::new(Mutex::new(Vec::new()));
+ let received = observed.clone();
+ let both_uploads = Arc::new(tokio::sync::Barrier::new(2));
+ let web = FakeServer::start(move |request| {
+ let received = received.clone();
+ let both_uploads = both_uploads.clone();
+ async move {
+ both_uploads.wait().await;
+ let authorization = request.headers()["authorization"]
+ .to_str()
+ .unwrap()
+ .to_owned();
+ let bytes = to_bytes(request.into_body(), 64 * 1024).await.unwrap();
+ let payload: Value = serde_json::from_slice(&bytes).unwrap();
+ let attributes = payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]
+ .as_array()
+ .unwrap();
+ let metadata = attributes
+ .iter()
+ .find(|attribute| attribute["key"] == "langfuse.observation.metadata")
+ .unwrap();
+ let metadata: Value =
+ serde_json::from_str(metadata["value"]["stringValue"].as_str().unwrap()).unwrap();
+ received.lock().unwrap().push((
+ authorization,
+ metadata["project_id"].as_str().unwrap().to_owned(),
+ ));
+ response(200, "{}")
+ }
+ })
+ .await;
+ let telemetry = Telemetry::with_uploader(uploader(&web.url), 2, MAX_RETAINED_BYTES);
+ for project in ["project-a", "project-b"] {
+ let mut grant = grant().await;
+ grant.project_id = project.into();
+ grant.access_token = format!("token-{project}");
+ telemetry.record(grant, facts(project));
+ }
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(2))
+ .await;
+ let mut actual = observed.lock().unwrap().clone();
+ actual.sort();
+ assert_eq!(
+ actual,
+ vec![
+ ("Bearer token-project-a".into(), "project-a".into()),
+ ("Bearer token-project-b".into(), "project-b".into())
+ ]
+ );
+ assert_eq!(telemetry.0.stats.accepted.load(Ordering::Relaxed), 2);
+}
+
+#[tokio::test]
+async fn upload_response_failures_are_reported_without_retries() {
+ let grant = grant().await;
+ for (status, body, expected) in [
+ (500, "{}", Err(ExportError::Rejected)),
+ (
+ 200,
+ r#"{"partialSuccess":{"rejectedSpans":"1"}}"#,
+ Err(ExportError::Partial),
+ ),
+ (
+ 200,
+ r#"{"partialSuccess":{"rejectedSpans":1}}"#,
+ Err(ExportError::Partial),
+ ),
+ (
+ 200,
+ r#"{"partialSuccess":{"rejectedSpans":"bad"}}"#,
+ Err(ExportError::Response),
+ ),
+ (200, "[]", Err(ExportError::Response)),
+ (200, "not json", Err(ExportError::Response)),
+ (200, r#"{"partialSuccess":{"rejectedSpans":"0"}}"#, Ok(())),
+ ] {
+ let web = FakeServer::start(move |_| async move { response(status, body) }).await;
+ assert_eq!(
+ uploader(&web.url).export(&grant, &[json!({})]).await,
+ expected
+ );
+ assert_eq!(web.calls(), 1);
+ }
+ // Streaming without Content-Length must obey the response bound too.
+ let web = FakeServer::start(|_| async move {
+ let chunks = futures_util::stream::iter([Ok::<_, std::io::Error>(vec![b' '; 65 * 1024])]);
+ response(200, Body::from_stream(chunks))
+ })
+ .await;
+ assert_eq!(
+ uploader(&web.url).export(&grant, &[json!({})]).await,
+ Err(ExportError::Response)
+ );
+ assert_eq!(web.calls(), 1);
+}
+
+#[tokio::test]
+async fn redirects_do_not_forward_the_ingestion_credentials() {
+ let destination = FakeServer::start(|_| async { response(200, "{}") }).await;
+ let location = destination.url.clone();
+ let web = FakeServer::start(move |_| {
+ let location = location.clone();
+ async move {
+ Response::builder()
+ .status(307)
+ .header("location", location)
+ .body(Body::empty())
+ .unwrap()
+ }
+ })
+ .await;
+ assert_eq!(
+ uploader(&web.url)
+ .export(&grant().await, &[json!({})])
+ .await,
+ Err(ExportError::Rejected)
+ );
+ assert_eq!(web.calls(), 1);
+ assert_eq!(destination.calls(), 0);
+}
+
+#[tokio::test]
+async fn expired_grants_and_oversized_payloads_never_reach_the_network() {
+ let web = FakeServer::start(|_| async { response(200, "{}") }).await;
+ let uploader = uploader(&web.url);
+ let mut expired = grant().await;
+ expired.expires_at = 1;
+ assert_eq!(
+ uploader.export(&expired, &[json!({})]).await,
+ Err(ExportError::Expired)
+ );
+ let oversized = json!({"oversized": "x".repeat(8 * 1024 * 1024)});
+ assert_eq!(
+ uploader.export(&grant().await, &[oversized]).await,
+ Err(ExportError::Payload)
+ );
+ assert_eq!(web.calls(), 0);
+}
+
+#[tokio::test]
+async fn upload_capacity_and_shutdown_deadline_bound_hanging_delivery() {
+ let started = Arc::new(Notify::new());
+ let received = started.clone();
+ let web = FakeServer::start(move |_| {
+ let received = received.clone();
+ async move {
+ received.notify_one();
+ std::future::pending::<()>().await;
+ response(200, "{}")
+ }
+ })
+ .await;
+ let telemetry = Telemetry::with_uploader(uploader(&web.url), 1, MAX_RETAINED_BYTES);
+ telemetry.record(grant().await, facts("project-1"));
+ tokio::time::timeout(Duration::from_secs(2), started.notified())
+ .await
+ .unwrap();
+ telemetry.record(grant().await, facts("project-1"));
+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 1);
+ telemetry.shutdown(Instant::now()).await;
+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 2);
+ assert_eq!(telemetry.0.capacity.available_permits(), 1);
+ assert_eq!(telemetry.0.bytes.available_permits(), MAX_RETAINED_BYTES);
+ telemetry.record(grant().await, facts("project-1"));
+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 3);
+ assert_eq!(web.calls(), 1);
+}
+
+#[tokio::test]
+async fn retained_byte_budget_is_released_after_successful_drain() {
+ let started = Arc::new(Notify::new());
+ let release = Arc::new(Notify::new());
+ let received = started.clone();
+ let gate = release.clone();
+ let web = FakeServer::start(move |_| {
+ let received = received.clone();
+ let gate = gate.clone();
+ async move {
+ received.notify_one();
+ gate.notified().await;
+ response(200, "{}")
+ }
+ })
+ .await;
+ let grant = grant().await;
+ let budget = serde_json::to_vec(&facts("project-1")).unwrap().len()
+ + grant.access_token.len()
+ + grant.project_id.len();
+ let telemetry = Telemetry::with_uploader(uploader(&web.url), 2, budget);
+ telemetry.record(grant, facts("project-1"));
+ tokio::time::timeout(Duration::from_secs(2), started.notified())
+ .await
+ .unwrap();
+ assert_eq!(telemetry.0.bytes.available_permits(), 0);
+ let second = DeliveryContext::from_resolved(&resolved_request_context("provider-secret").await);
+ telemetry.record(second, facts("project-1"));
+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 1);
+ release.notify_one();
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(2))
+ .await;
+ assert_eq!(telemetry.0.bytes.available_permits(), budget);
+ assert_eq!(telemetry.0.capacity.available_permits(), 2);
+ assert_eq!(telemetry.0.stats.accepted.load(Ordering::Relaxed), 1);
+ assert_eq!(web.calls(), 1);
+}
+
+#[tokio::test]
+async fn oversized_facts_are_dropped_before_task_admission() {
+ let web = FakeServer::start(|_| async { response(200, "{}") }).await;
+ let telemetry = Telemetry::with_uploader(uploader(&web.url), 1, MAX_RETAINED_BYTES);
+ let mut oversized = facts("project-1");
+ oversized.metadata["oversized"] = json!("x".repeat(MAX_FACT_BYTES));
+ telemetry.record(grant().await, oversized);
+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 1);
+ assert_eq!(telemetry.0.capacity.available_permits(), 1);
+ assert_eq!(telemetry.0.bytes.available_permits(), MAX_RETAINED_BYTES);
+ telemetry
+ .shutdown(Instant::now() + Duration::from_secs(2))
+ .await;
+ assert_eq!(web.calls(), 0);
+}
diff --git a/packages/shared/scripts/seeder/seed-postgres.ts b/packages/shared/scripts/seeder/seed-postgres.ts
index d9a78bb796..6e7f61312b 100644
--- a/packages/shared/scripts/seeder/seed-postgres.ts
+++ b/packages/shared/scripts/seeder/seed-postgres.ts
@@ -557,7 +557,7 @@ async function seedAiGateway(params: {
create: {
organizationId: params.organizationId,
defaultIngestionProjectId: ingestionProjectId,
- ingestionMode: "USAGE",
+ ingestionMode: "FULL",
},
update: {
defaultIngestionProjectId: ingestionProjectId,
diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.ts
index fb511e8f8a..549dee4acd 100644
--- a/worker/src/services/ClickhouseWriter/index.ts
+++ b/worker/src/services/ClickhouseWriter/index.ts
@@ -90,8 +90,6 @@ export class ClickhouseWriter {
this.isIntervalFlushInProgress = true;
- logger.debug("Flush interval elapsed, flushing all queues...");
-
this.flushAll().finally(() => {
this.isIntervalFlushInProgress = false;
});