🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): upload finalized generations to Langfuse (#17470)
Hassieb Pakzad6adfa3f75d49
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lockindex de2918a800..3a4d768c93 100644--- a/ai-gateway/Cargo.lock+++ b/ai-gateway/Cargo.lock@@ -7,6 +7,7 @@ name = "ai-gateway"version = "0.1.0"dependencies = ["axum",+ "chrono","futures-util","hmac","http-body-util",@@ -18,6 +19,7 @@ dependencies = ["tower","tracing","tracing-subscriber",+ "uuid",][[package]]@@ -26,6 +28,12 @@ version = "1.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"+[[package]]+name = "autocfg"+version = "1.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"+[[package]]name = "axum"version = "0.8.9"@@ -140,6 +148,15 @@ dependencies = ["rand_core",]+[[package]]+name = "chrono"+version = "0.4.45"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"+dependencies = [+ "num-traits",+]+[[package]]name = "cmov"version = "0.5.4"@@ -639,6 +656,15 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "num-traits"+version = "0.2.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"+dependencies = [+ "autocfg",+]+[[package]]name = "once_cell"version = "1.21.4"@@ -1355,6 +1381,17 @@ version = "1.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"+[[package]]+name = "uuid"+version = "1.26.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce"+dependencies = [+ "getrandom 0.4.3",+ "js-sys",+ "wasm-bindgen",+]+[[package]]name = "valuable"version = "0.1.1"diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.tomlindex 316aa4f509..c56efb94eb 100644--- a/ai-gateway/Cargo.toml+++ b/ai-gateway/Cargo.toml@@ -7,6 +7,7 @@ license = "MIT"[dependencies]axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] }+chrono = { version = "0.4", default-features = false, features = ["std"] }hmac = "0.13"http-body-util = "0.1"futures-util = "0.3"@@ -17,6 +18,7 @@ sha2 = "0.11"tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }tracing = "0.1.44"tracing-subscriber = { version = "0.3.23", default-features = false, features = ["fmt", "json"] }+uuid = { version = "1", features = ["v4"] }[dev-dependencies]tokio = { version = "1.53.1", features = ["io-util"] }diff --git a/ai-gateway/README.md b/ai-gateway/README.mdindex 336e72e09e..263bfa86e7 100644--- a/ai-gateway/README.md+++ b/ai-gateway/README.md@@ -3,8 +3,9 @@A standalone Rust gateway for `POST /openai/v1/responses`. Web resolves the gatewaykey to a trusted provider connection; Rust relays native JSON or SSE withoutrewriting provider bytes. A bounded capture layer captures request/response facts and-logs the finalized record at debug level. Customer telemetry is not exported-yet. Building and testing need no real Web, database or provider credentials.+logs the finalized record at debug level. Each finalized execution is immediately+uploaded as a Langfuse generation through OTLP. Building and testing need no real+Web, database or provider credentials.## Run locally@@ -124,7 +125,7 @@ response = client.responses.create(model="gpt-4.1-mini", input="Say hello")print(response.output_text)```-The gateway forwards to the official OpenAI Responses endpoint only. It does not+The gateway forwards inference to the official OpenAI Responses endpoint only. It does notretry, follow redirects, or accept client routing overrides. Request parsing isbest-effort capture only; Web still selects the provider connection.Provider errors retain their status and body. Gateway errors use an OpenAI-style@@ -158,6 +159,50 @@ timing/version/rate-limitheaders are retained. Cookies, routing overrides, gateway/ingestion credentials,hop-by-hop headers and upstream framing are excluded.+## Langfuse uploads++When inference is configured, each finalized execution starts one background POST+to the Web base URL's `/api/public/otel/v1/traces` endpoint. The client response and+provider admission never wait for ingestion. There is no batching, waiting queue,+retry, or durable delivery. A successful upload acknowledges ingestion acceptance;+storage and cost processing still happen asynchronously in Langfuse.++The original resolver-issued project grant authenticates the upload, together with+a fresh gateway HMAC signature over that grant. The uploader uses the existing Web+URL and service key, preserves deployment prefixes, disables redirects and proxies,+and never sends the provider credential or the client's gateway key to ingestion.+Ingestion JWTs stay outside serializable capture facts and debug logs. Expired grants+are dropped without re-resolving the execution. Requests explicitly select v4 ingestion.++Each execution becomes one root generation with generated trace/observation IDs,+actual/requested model, model parameters, native usage, and trusted key/connection+attribution. Full mode includes the captured native input/output; usage mode omits+content. First-byte timing supplies Langfuse's completion-start time as the gateway's+first-token approximation. Relay outcome, provider status and capture completeness+remain separate metadata fields. The exporter projects native OpenAI usage into the+receiver's supported shape for pricing and preserves the untouched usage object as+`native_usage` metadata. Missing usage is not reported as zero.++Provisional upload limits are 32 concurrent tasks, 4 MiB serialized facts per record,+16 MiB total retained serialized-fact/credential bytes, 8 MiB encoded payloads, and+64 KiB ingestion responses. Serialization and mapping have additional bounded memory+overhead; these byte budgets are not an RSS limit. Uploads have a two-second connect+timeout and a five-second total timeout, shortened to the grant's remaining lifetime.+Capacity exhaustion, oversized payloads, auth/HTTP errors, rejected OTLP spans, and+transport failures are reported without changing inference results. Sanitized logs+record failures/drops; shutdown reports accepted, failed and dropped counts.++SIGTERM marks the gateway unready, drains inference, then finishes uploads within+the remaining shared shutdown budget. At the deadline, unfinished uploads are+cancelled and counted as drops. Process crashes or forced shutdown can lose telemetry;+these records are not a durable accounting ledger.++`telemetry/mod.rs` owns admission and task lifecycle, `telemetry/mapping.rs` maps facts,+and `telemetry/otlp.rs` owns encoding and HTTP delivery. The uploader already accepts+a span collection; future project batching can replace immediate scheduling while+retaining each execution's original attribution/content policy and selecting a valid+compatible grant. Capture and provider byte forwarding do not need to change.+## Response captureSet `LANGFUSE_LOG_LEVEL=debug` to print one `gateway response captured` event at@@ -190,8 +235,8 @@ output array at EOF. Missing usage remains null, rather than invented zeros.`outcome` describes the relay (`eof`, `cancelled`, `timeout`, `transport_error`);`provider_status` is separate. For example, a completed provider response canstill end with downstream cancellation. `first_byte_ms` measures the first body-bytes observed by the gateway, not first-token latency. No completion-start time-is fabricated from completed items.+bytes observed by the gateway. Telemetry uses this as the first-token approximation+for Langfuse completion-start time; it does not wait for completed output items.`input_complete` means the full native request was captured; it is false whencontent is omitted in usage mode. `output_complete` means response inspection@@ -205,10 +250,10 @@ fully captured response, and downstream cancellation does not erase completed caThe implementation separates shared `ExecutionCapture` lifecycle/timing, the`OpenAiResponsesCapture` adapter and bounded `SseDecoder`. A private `ProtocolCapture`enum dispatches to the adapter. Finalization hands an owned `InferenceFacts` record-to `telemetry::record`, which currently only pretty-prints at debug level. Another-provider can supply native facts through the same interface without changing the-relay or final sink. No Anthropic adapter or Langfuse observation/span mapping is-implemented yet. Capture runs independently of log level; only emission is gated.+to telemetry for debug logging and immediate upload. Another provider can supply+native facts through the same interface without changing the relay. The current+usage mapping targets OpenAI Responses; no Anthropic adapter is implemented yet.+Capture and upload run independently of log level; debug emission alone is gated.Capture is limited to 1 MiB request inspection, 1 MiB JSON/SSE event inspection,1 MiB retained output and 256 output items per execution. The active-request limitdiff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rsindex 2ceda4752e..03683f4151 100644--- a/ai-gateway/src/capture/mod.rs+++ b/ai-gateway/src/capture/mod.rs@@ -13,7 +13,7 @@ use crate::{resolution::{IngestionMode, MetadataValue, ResolvedRequestContext},telemetry,};-use facts::ProviderFacts;+pub(crate) use facts::ProviderFacts;pub(crate) use facts::{InferenceFacts, RelayOutcome};use openai_responses::OpenAiResponsesCapture;@@ -61,6 +61,7 @@ pub(crate) struct ExecutionCapture {first_byte_ms: Option<u128>,http_status: Option<u16>,metadata: Value,+ delivery: Option<(telemetry::Telemetry, telemetry::DeliveryContext)>,}impl ExecutionCapture {@@ -106,9 +107,21 @@ impl ExecutionCapture {"key_metadata": key_metadata,"ingestion_mode": if full { "full" } else { "usage" },}),+ delivery: None,}}+ pub fn deliver_to(+ &mut self,+ telemetry: telemetry::Telemetry,+ context: &ResolvedRequestContext,+ ) {+ self.delivery = Some((+ telemetry,+ telemetry::DeliveryContext::from_resolved(context),+ ));+ }+pub fn response(&mut self, status: u16, headers: &HeaderMap) {if let Some(protocol) = &mut self.protocol {self.http_status = Some(status);@@ -137,7 +150,7 @@ impl ExecutionCapture {return;};let (api_format, inference) = protocol.into_facts();- telemetry::record(InferenceFacts {+ let facts = InferenceFacts {api_format,start_time_unix_ms: self.start_time_unix_ms,duration_ms: self.started.elapsed().as_millis(),@@ -146,7 +159,11 @@ impl ExecutionCapture {metadata: std::mem::take(&mut self.metadata),outcome,inference,- });+ };+ telemetry::debug_record(&facts);+ if let Some((telemetry, context)) = self.delivery.take() {+ telemetry.record(context, facts);+ }}}diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rsindex fce29bf6bc..37fba9035c 100644--- a/ai-gateway/src/inference.rs+++ b/ai-gateway/src/inference.rs@@ -16,6 +16,7 @@ pub struct InferenceService {control_plane: ControlPlaneClient,provider: OpenAiProvider,resolution_capacity: Semaphore,+ telemetry: Option<crate::telemetry::Telemetry>,}pub(crate) enum RequestPreparationError {@@ -36,14 +37,21 @@ impl InferenceService {if !(1..=Semaphore::MAX_PERMITS).contains(&max_concurrent_resolutions) {return Err(ResolutionError::Configuration);}+ let telemetry = crate::telemetry::Telemetry::new(&config)?;Ok(Self {control_plane: ControlPlaneClient::new(config)?,provider: OpenAiProvider::new(max_active_requests)- .map_err(|_| ResolutionError::Configuration)?,+ .map_err(|_| ResolutionError::Configuration)?+ .with_telemetry(telemetry.clone()),resolution_capacity: Semaphore::new(max_concurrent_resolutions),+ telemetry: Some(telemetry),})}+ pub fn telemetry(&self) -> Option<crate::telemetry::Telemetry> {+ self.telemetry.clone()+ }+/// Authenticate with a separate bounded budget before reserving execution capacity.pub(crate) async fn resolve_and_admit(&self,@@ -86,6 +94,7 @@ impl InferenceService {control_plane,provider,resolution_capacity: Semaphore::new(resolutions),+ telemetry: None,}}}diff --git a/ai-gateway/src/lib.rs b/ai-gateway/src/lib.rsindex ce4f00cec5..1d06512e43 100644--- a/ai-gateway/src/lib.rs+++ b/ai-gateway/src/lib.rs@@ -5,7 +5,7 @@ pub mod inference;pub mod providers;pub mod resolution;pub mod server;-mod telemetry;+pub mod telemetry;mod transport;#[cfg(test)]diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rsindex 3d4873dabd..94c87f689f 100644--- a/ai-gateway/src/main.rs+++ b/ai-gateway/src/main.rs@@ -39,6 +39,7 @@ async fn run() -> Result<(), Box<dyn Error>> {}).transpose()?;let inference_enabled = inference.is_some();+ let telemetry = inference.as_ref().and_then(InferenceService::telemetry);let state = if inference_enabled {GatewayLifecycleState::default()} else {@@ -48,7 +49,24 @@ async fn run() -> Result<(), Box<dyn Error>> {let shutdown = shutdown_signal()?;let listener = bind_listener(config.listen_address, config.auto_increment_listen_port).await?;tracing::info!(address = %listener.local_addr()?, inference_enabled, "gateway listening");- server::serve(listener, app, state, shutdown, config.shutdown_timeout).await?;+ let result = server::serve(+ listener,+ app,+ state.clone(),+ shutdown,+ config.shutdown_timeout,+ )+ .await;+ if let Some(telemetry) = telemetry {+ telemetry+ .shutdown(+ state+ .drain_deadline()+ .unwrap_or_else(tokio::time::Instant::now),+ )+ .await;+ }+ result?;tracing::info!("gateway stopped");Ok(())}diff --git a/ai-gateway/src/providers/openai.rs b/ai-gateway/src/providers/openai.rsindex 71a6568056..7b05775e35 100644--- a/ai-gateway/src/providers/openai.rs+++ b/ai-gateway/src/providers/openai.rs@@ -48,6 +48,7 @@ pub struct OpenAiProvider {client: Client,capacity: Arc<Semaphore>,limits: ProviderLimits,+ telemetry: Option<crate::telemetry::Telemetry>,#[cfg(test)]endpoint: String,}@@ -85,11 +86,17 @@ impl OpenAiProvider {client,capacity: Arc::new(Semaphore::new(limits.active)),limits,+ telemetry: None,#[cfg(test)]endpoint: "https://api.openai.com/v1/responses".to_owned(),})}+ pub(crate) fn with_telemetry(mut self, telemetry: crate::telemetry::Telemetry) -> Self {+ self.telemetry = Some(telemetry);+ self+ }+/// Reserve capacity for an authenticated request before reading its body.////// # Errors@@ -129,6 +136,9 @@ impl OpenAiProvider {.map_err(|_| ProviderError::Configuration)?;authorization.set_sensitive(true);let mut capture = ExecutionCapture::openai_responses(&context, headers, &body);+ if let Some(telemetry) = &self.telemetry {+ capture.deliver_to(telemetry.clone(), &context);+ }let response = tokio::time::timeout_at(permit.deadlinediff --git a/ai-gateway/src/providers/openai/tests.rs b/ai-gateway/src/providers/openai/tests.rsindex 8c1d502e72..7cbb25a600 100644--- a/ai-gateway/src/providers/openai/tests.rs+++ b/ai-gateway/src/providers/openai/tests.rs@@ -331,3 +331,209 @@ async fn deadlines_bound_headers_and_stalled_bodies_without_exposing_transport_dassert!(!error.to_string().contains("provider-secret"));assert!(relay.try_admit().is_ok());}++#[tokio::test]+async fn completed_json_and_sse_upload_once_without_waiting_for_ingestion() {+ use crate::{+ resolution::ControlPlaneConfig, telemetry::Telemetry,+ test_support::resolved_request_context_with_mode,+ };+ use serde_json::Value;++ for (content_type, native, sink_status) in [+ (+ "application/json",+ r#"{"id":"resp-1","model":"actual","status":"completed","output":[],"usage":{"input_tokens":2,"output_tokens":1,"total_tokens":3}}"#,+ 200,+ ),+ (+ "text/event-stream",+ "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp-1\",\"model\":\"actual\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":2,\"output_tokens\":1,\"total_tokens\":3}}}\n\n",+ 503,+ ),+ ] {+ let (sent, mut received) = tokio::sync::mpsc::channel(2);+ let release = Arc::new(Notify::new());+ let sink_release = release.clone();+ let sink = FakeServer::start(move |request| {+ let sent = sent.clone();+ let release = sink_release.clone();+ async move {+ let bytes = to_bytes(request.into_body(), 65536).await.unwrap();+ sent.send(serde_json::from_slice::<Value>(&bytes).unwrap())+ .await+ .unwrap();+ release.notified().await;+ Response::builder()+ .status(sink_status)+ .body(Body::from("{}"))+ .unwrap()+ }+ })+ .await;+ let telemetry =+ Telemetry::new(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap()).unwrap();+ let upstream = FakeServer::start(move |_| async move {+ Response::builder()+ .header("content-type", content_type)+ .body(Body::from(native))+ .unwrap()+ })+ .await;+ let relay = provider(&upstream, 1).with_telemetry(telemetry.clone());+ let context = resolved_request_context_with_mode("provider-secret", "full").await;+ let response = relay+ .forward(+ relay.try_admit().unwrap(),+ context,+ &HeaderMap::new(),+ Bytes::from_static(br#"{"model":"requested","input":"hello"}"#),+ )+ .await+ .unwrap();+ // Neither the last byte nor the provider permit waits for the sink response.+ assert_eq!(+ tokio::time::timeout(Duration::from_secs(1), to_bytes(response.into_body(), 4096))+ .await+ .unwrap()+ .unwrap(),+ native+ );+ assert!(relay.try_admit().is_ok());+ let payload = tokio::time::timeout(Duration::from_secs(1), received.recv())+ .await+ .unwrap()+ .unwrap();+ assert_completed_upload(&payload);+ release.notify_one();+ telemetry+ .shutdown(Instant::now() + Duration::from_secs(1))+ .await;+ assert!(received.try_recv().is_err());+ assert_eq!(sink.calls(), 1);+ assert_eq!(upstream.calls(), 1);+ }+}++#[tokio::test]+async fn cancelled_and_timed_out_executions_upload_after_provider_context_is_released() {+ use crate::{resolution::ControlPlaneConfig, telemetry::Telemetry};+ use serde_json::Value;++ for cancelled in [true, false] {+ let (sent, mut received) = tokio::sync::mpsc::channel(2);+ let sink = FakeServer::start(move |request| {+ let sent = sent.clone();+ async move {+ assert_eq!(+ request.headers()[header::AUTHORIZATION],+ "Bearer private-ingestion-token"+ );+ let bytes = to_bytes(request.into_body(), 65536).await.unwrap();+ sent.send(serde_json::from_slice::<Value>(&bytes).unwrap())+ .await+ .unwrap();+ Response::new(Body::from("{}"))+ }+ })+ .await;+ let telemetry =+ Telemetry::new(&ControlPlaneConfig::new(&sink.url, "service-key").unwrap()).unwrap();+ let dropped = Arc::new(Notify::new());+ let upstream = stalled_provider(dropped).await;+ let relay = OpenAiProvider::for_test(+ upstream.url.clone(),+ ProviderLimits {+ active: 1,+ execution_timeout: Duration::from_millis(150),+ ..ProviderLimits::default()+ },+ )+ .with_telemetry(telemetry.clone());+ let context = resolved_request_context("provider-secret").await;+ let response = relay+ .forward(+ relay.try_admit().unwrap(),+ context,+ &HeaderMap::new(),+ Bytes::from_static(br#"{"model":"requested","input":"prompt-canary"}"#),+ )+ .await+ .unwrap();+ let mut body = response.into_body().into_data_stream();+ assert_eq!(body.next().await.unwrap().unwrap(), "first");+ assert_eq!(sink.calls(), 0);+ if cancelled {+ drop(body);+ } else {+ assert!(body.next().await.unwrap().is_err());+ drop(body);+ }+ let payload = tokio::time::timeout(Duration::from_secs(1), received.recv())+ .await+ .unwrap()+ .unwrap();+ assert!(relay.try_admit().is_ok());+ assert!(!payload.to_string().contains("prompt-canary"));+ let attrs = payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]+ .as_array()+ .unwrap();+ let metadata: Value = serde_json::from_str(+ attrs+ .iter()+ .find(|a| a["key"] == "langfuse.observation.metadata")+ .unwrap()["value"]["stringValue"]+ .as_str()+ .unwrap(),+ )+ .unwrap();+ assert_eq!(+ metadata["relay_outcome"],+ if cancelled { "cancelled" } else { "timeout" }+ );+ telemetry+ .shutdown(Instant::now() + Duration::from_secs(1))+ .await;+ assert!(received.try_recv().is_err());+ assert_eq!(sink.calls(), 1);+ }+}++fn assert_completed_upload(payload: &serde_json::Value) {+ use serde_json::{Value, json};+ let span = &payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0];+ let attrs = span["attributes"].as_array().unwrap();+ assert!(+ attrs+ .iter()+ .any(|a| a["key"] == "langfuse.observation.model.name"+ && a["value"]["stringValue"] == "actual")+ );+ assert!(+ attrs+ .iter()+ .any(|a| a["key"] == "langfuse.observation.completion_start_time")+ );+ let metadata: Value = serde_json::from_str(+ attrs+ .iter()+ .find(|a| a["key"] == "langfuse.observation.metadata")+ .unwrap()["value"]["stringValue"]+ .as_str()+ .unwrap(),+ )+ .unwrap();+ assert_eq!(metadata["relay_outcome"], "eof");+ assert_eq!(+ metadata["native_usage"],+ json!({"input_tokens":2,"output_tokens":1,"total_tokens":3})+ );+ for secret in [+ "provider-secret",+ "private-ingestion-token",+ "gateway-secret",+ "service-key",+ ] {+ assert!(!payload.to_string().contains(secret));+ }+}diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rsindex 83d1c61f19..fe5adfc95a 100644--- a/ai-gateway/src/resolution/mod.rs+++ b/ai-gateway/src/resolution/mod.rs@@ -1,6 +1,6 @@//! Resolve credentials through trusted Web before any provider execution.mod contracts;-mod signing;+pub(crate) mod signing;pub use contracts::{ApiFormat, IngestionGrant, IngestionMode, MetadataValue, ProviderConnection,@@ -25,7 +25,7 @@ struct ResolutionLimits {/// Operator-supplied Web base URL and the existing gateway/Web service key.pub struct ControlPlaneConfig {- url: Url,+ web_url: Url,service_key: String,limits: ResolutionLimits,}@@ -38,7 +38,7 @@ impl ControlPlaneConfig {/// base URL. URLs require HTTPS (HTTP is allowed on loopback), with no userinfo,/// query, or fragment.pub fn new(web_url: &str, service_key: &str) -> Result<Self, ResolutionError> {- let mut url = Url::parse(web_url).map_err(|_| ResolutionError::Configuration)?;+ let url = Url::parse(web_url).map_err(|_| ResolutionError::Configuration)?;let loopback = url.host_str().is_some_and(|host| {host == "localhost"|| host@@ -56,10 +56,8 @@ impl ControlPlaneConfig {{return Err(ResolutionError::Configuration);}- let resolve_path = format!("{}{RESOLVE_PATH}", url.path().trim_end_matches('/'));- url.set_path(&resolve_path);Ok(Self {- url,+ web_url: url,service_key: service_key.to_owned(),limits: ResolutionLimits {timeout: Duration::from_secs(5),@@ -67,6 +65,16 @@ impl ControlPlaneConfig {},})}++ pub(crate) fn endpoint(&self, path: &str) -> Url {+ let mut url = self.web_url.clone();+ url.set_path(&format!("{}{path}", url.path().trim_end_matches('/')));+ url+ }++ pub(crate) fn service_key(&self) -> &str {+ &self.service_key+ }}/// Reuses the HTTP connection pool; credentials belong exclusively to each request.@@ -154,7 +162,7 @@ impl ControlPlaneClient {.map_err(|_| ResolutionError::Configuration)?;let mut response = self.client- .post(self.config.url.clone())+ .post(self.config.endpoint(RESOLVE_PATH)).header(AUTHORIZATION, credential).header("langfuse-gateway-authorization", signature).header(CONTENT_TYPE, "application/json")diff --git a/ai-gateway/src/resolution/signing.rs b/ai-gateway/src/resolution/signing.rsindex 0e5303f4a8..776314d920 100644--- a/ai-gateway/src/resolution/signing.rs+++ b/ai-gateway/src/resolution/signing.rs@@ -2,8 +2,8 @@ use hmac::{Hmac, KeyInit, Mac};use sha2::{Digest, Sha256};use std::fmt::Write;-pub(super) fn authorization(service_key: &str, gateway_key: &str, timestamp: u64) -> String {- let key_hash = lowercase_hex(&Sha256::digest(gateway_key.as_bytes()));+pub(crate) fn authorization(service_key: &str, credential: &str, timestamp: u64) -> String {+ let key_hash = lowercase_hex(&Sha256::digest(credential.as_bytes()));let payload = format!("gateway-web-v1\n{timestamp}\n{key_hash}");// HMAC accepts keys of any length.let mut mac = Hmac::<Sha256>::new_from_slice(service_key.as_bytes()).expect("valid HMAC key");diff --git a/ai-gateway/src/server.rs b/ai-gateway/src/server.rsindex e0a65e9d89..70e82dcdf8 100644--- a/ai-gateway/src/server.rs+++ b/ai-gateway/src/server.rs@@ -2,7 +2,7 @@ use std::{future::{Future, IntoFuture},io,sync::{- Arc,+ Arc, Mutex,atomic::{AtomicBool, Ordering},},time::Duration,@@ -16,9 +16,21 @@ use tokio::{net::TcpListener, sync::oneshot};pub struct GatewayLifecycleState {ready: Arc<AtomicBool>,disabled: bool,+ drain_deadline: Arc<Mutex<Option<tokio::time::Instant>>>,}impl GatewayLifecycleState {+ /// Deadline shared by connection draining and telemetry delivery.+ ///+ /// # Panics+ /// Panics if the lifecycle lock was poisoned.+ pub fn drain_deadline(&self) -> Option<tokio::time::Instant> {+ *self+ .drain_deadline+ .lock()+ .expect("gateway drain lock poisoned")+ }+pub fn is_ready(&self) -> bool {!self.disabled && self.ready.load(Ordering::Acquire)}@@ -67,6 +79,9 @@ async fn readiness(State(state): State<GatewayLifecycleState>) -> (StatusCode, J/// # Errors/// Returns the server's I/O error, or [`io::ErrorKind::TimedOut`] if connections/// do not finish draining within `shutdown_timeout`.+///+/// # Panics+/// Panics if the lifecycle lock was poisoned.pub async fn serve(listener: TcpListener,app: Router,@@ -80,6 +95,11 @@ pub async fn serve(.with_graceful_shutdown(async move {shutdown.await;shutdown_state.ready.store(false, Ordering::Release);+ *shutdown_state+ .drain_deadline+ .lock()+ .expect("gateway drain lock poisoned") =+ Some(tokio::time::Instant::now() + shutdown_timeout);tracing::info!("gateway draining");let _ = draining_tx.send(());})diff --git a/ai-gateway/src/telemetry.rs b/ai-gateway/src/telemetry.rsdeleted file mode 100644index bedc5daaac..0000000000--- a/ai-gateway/src/telemetry.rs+++ /dev/null@@ -1,7 +0,0 @@-//! Final facts delivery. A Langfuse exporter can replace this debug sink later.-use crate::capture::InferenceFacts;--pub(crate) fn record(facts: InferenceFacts) {- // Full mode intentionally includes native request and completed output content.- tracing::debug!(capture = %format_args!("{:#}", serde_json::to_value(facts).expect("captured facts must serialize")), "gateway response captured");-}diff --git a/ai-gateway/src/telemetry/mapping.rs b/ai-gateway/src/telemetry/mapping.rsnew file mode 100644index 0000000000..9b61012e3a--- /dev/null+++ b/ai-gateway/src/telemetry/mapping.rs@@ -0,0 +1,310 @@+//! Pure mapping from finalized execution facts to a Langfuse generation span.+use chrono::{DateTime, SecondsFormat};+use serde_json::{Map, Value, json};++use crate::capture::{InferenceFacts, RelayOutcome};++pub(super) fn span(facts: InferenceFacts, trace_id: &str, observation_id: &str) -> Value {+ let full = facts.metadata.get("ingestion_mode").and_then(Value::as_str) == Some("full");+ let (level, message) = observation_status(&facts);+ let inference = facts.inference;+ let mut metadata = facts.metadata.as_object().cloned().unwrap_or_default();+ metadata.extend([+ ("api_format".into(), json!(facts.api_format)),+ ("relay_outcome".into(), json!(facts.outcome)),+ ("http_status".into(), json!(facts.http_status)),+ ("first_byte_ms".into(), json!(facts.first_byte_ms)),+ ("requested_model".into(), json!(inference.requested_model)),+ (+ "provider_response_id".into(),+ json!(inference.provider_response_id),+ ),+ (+ "provider_request_id".into(),+ json!(inference.provider_request_id),+ ),+ ("provider_status".into(), json!(inference.provider_status)),+ ("input_complete".into(), json!(inference.input_complete)),+ ("output_complete".into(), json!(inference.output_complete)),+ ("capture_complete".into(), json!(inference.capture_complete)),+ ]);+ let mut attributes = vec![attribute("langfuse.observation.type", "generation")];+ if let Some(model) = inference.model.or(inference.requested_model) {+ attributes.push(attribute("langfuse.observation.model.name", model));+ }+ if !inference.model_parameters.is_empty() {+ attributes.push(attribute(+ "langfuse.observation.model.parameters",+ Value::Object(inference.model_parameters).to_string(),+ ));+ }+ if let Some(usage) = inference.usage_details {+ if let Some(projected) = openai_usage(&usage) {+ attributes.push(attribute(+ "langfuse.observation.usage_details",+ projected.to_string(),+ ));+ }+ metadata.insert("native_usage".into(), usage);+ }+ if full {+ if let Some(input) = inference.input {+ attributes.push(attribute("langfuse.observation.input", input.to_string()));+ }+ if let Some(output) = inference.output {+ attributes.push(attribute("langfuse.observation.output", output.to_string()));+ }+ }+ if let Some(completion_start) = facts+ .first_byte_ms+ .and_then(|elapsed| facts.start_time_unix_ms.checked_add(elapsed))+ .and_then(|timestamp| i64::try_from(timestamp).ok())+ .and_then(DateTime::from_timestamp_millis)+ {+ attributes.push(attribute(+ "langfuse.observation.completion_start_time",+ completion_start.to_rfc3339_opts(SecondsFormat::Millis, true),+ ));+ }+ attributes.push(attribute("langfuse.observation.level", level));+ if let Some(message) = message {+ attributes.push(attribute("langfuse.observation.status_message", message));+ }+ attributes.push(attribute(+ "langfuse.observation.metadata",+ Value::Object(metadata).to_string(),+ ));+ json!({+ "traceId": trace_id,+ "spanId": observation_id,+ "name": facts.api_format,+ "kind": 3,+ "startTimeUnixNano": facts.start_time_unix_ms.saturating_mul(1_000_000).to_string(),+ "endTimeUnixNano": facts.start_time_unix_ms.saturating_add(facts.duration_ms).saturating_mul(1_000_000).to_string(),+ "attributes": attributes,+ "status": { "code": if level == "ERROR" { 2 } else { 0 } },+ })+}++fn observation_status(facts: &InferenceFacts) -> (&'static str, Option<&'static str>) {+ let failure = if facts.http_status.is_some_and(|status| status >= 400) {+ Some("Provider HTTP error")+ } else if facts.inference.provider_status.as_deref() == Some("failed") {+ Some("Provider response failed")+ } else {+ match facts.outcome {+ RelayOutcome::Timeout => Some("Provider response timed out"),+ RelayOutcome::TransportError => Some("Provider transport error"),+ RelayOutcome::Eof | RelayOutcome::Cancelled => None,+ }+ };+ if let Some(message) = failure {+ ("ERROR", Some(message))+ } else if facts.outcome == RelayOutcome::Cancelled {+ ("WARNING", Some("Client cancelled the response"))+ } else if facts.inference.provider_status.as_deref() == Some("incomplete") {+ ("WARNING", Some("Provider response incomplete"))+ } else {+ ("DEFAULT", None)+ }+}++fn attribute(key: &str, value: impl Into<String>) -> Value {+ json!({"key": key, "value": {"stringValue": value.into()}})+}++/// The receiver's native `OpenAI` usage schema is strict at the top level, but+/// accepts new numeric detail counters. Keep the original payload in metadata.+fn openai_usage(usage: &Value) -> Option<Value> {+ let usage = usage.as_object()?;+ let mut projected = Map::new();+ for key in ["input_tokens", "output_tokens", "total_tokens"] {+ let value = usage.get(key).filter(|value| value.as_u64().is_some())?;+ projected.insert(key.into(), value.clone());+ }+ for key in ["input_tokens_details", "output_tokens_details"] {+ match usage.get(key) {+ Some(Value::Null) => {+ projected.insert(key.into(), Value::Null);+ }+ Some(Value::Object(details)) => {+ projected.insert(+ key.into(),+ Value::Object(+ details+ .iter()+ .filter(|(_, value)| value.is_null() || value.as_u64().is_some())+ .map(|(key, value)| (key.clone(), value.clone()))+ .collect(),+ ),+ );+ }+ _ => {}+ }+ }+ Some(Value::Object(projected))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn facts() -> InferenceFacts {+ let mut facts = InferenceFacts {+ api_format: "openai.responses",+ start_time_unix_ms: 1_735_689_600_000,+ duration_ms: 900,+ first_byte_ms: Some(125),+ http_status: Some(200),+ metadata: json!({"ingestion_mode": "full", "project_id": "project", "key_id": "key", "key_metadata": {"team": "search"}}),+ outcome: RelayOutcome::Eof,+ inference: crate::capture::ProviderFacts::default(),+ };+ facts.inference.model = Some("actual-model".into());+ facts.inference.requested_model = Some("requested-model".into());+ facts+ .inference+ .model_parameters+ .insert("service_tier".into(), json!("priority"));+ facts.inference.input = Some(json!({"input": "hello", "tools": [{"type": "function"}]}));+ facts.inference.output = Some(json!([{ "type": "message", "content": "world" }]));+ facts.inference.capture_complete = true;+ facts.inference.output_complete = true;+ facts.inference.input_complete = true;+ facts+ }++ fn attributes(span: &Value) -> Map<String, Value> {+ span["attributes"]+ .as_array()+ .unwrap()+ .iter()+ .map(|attribute| {+ (+ attribute["key"].as_str().unwrap().into(),+ attribute["value"]["stringValue"].clone(),+ )+ })+ .collect()+ }++ fn metadata(attributes: &Map<String, Value>) -> Value {+ serde_json::from_str(+ attributes["langfuse.observation.metadata"]+ .as_str()+ .unwrap(),+ )+ .unwrap()+ }++ #[test]+ fn full_generation_preserves_native_content_attribution_and_first_byte_time() {+ let span = span(+ facts(),+ "0123456789abcdef0123456789abcdef",+ "0123456789abcdef",+ );+ assert_eq!(span["traceId"], "0123456789abcdef0123456789abcdef");+ assert_eq!(span["spanId"], "0123456789abcdef");+ assert!(span.get("parentSpanId").is_none());+ assert_eq!(span["startTimeUnixNano"], "1735689600000000000");+ assert_eq!(span["endTimeUnixNano"], "1735689600900000000");+ let attrs = attributes(&span);+ assert_eq!(attrs["langfuse.observation.type"], "generation");+ assert_eq!(attrs["langfuse.observation.model.name"], "actual-model");+ assert_eq!(+ attrs["langfuse.observation.completion_start_time"],+ "2025-01-01T00:00:00.125Z"+ );+ assert_eq!(+ serde_json::from_str::<Value>(attrs["langfuse.observation.input"].as_str().unwrap())+ .unwrap()["tools"][0]["type"],+ "function"+ );+ assert!(attrs.contains_key("langfuse.observation.output"));+ let metadata = metadata(&attrs);+ assert_eq!(metadata["project_id"], "project");+ assert_eq!(metadata["key_metadata"]["team"], "search");+ assert_eq!(metadata["requested_model"], "requested-model");+ assert_eq!(metadata["capture_complete"], true);+ }++ #[test]+ fn usage_mode_omits_content_and_missing_usage_and_uses_requested_model() {+ let mut facts = facts();+ facts.metadata["ingestion_mode"] = json!("usage");+ facts.inference.model = None;+ facts.first_byte_ms = None;+ let attrs = attributes(&span(facts, "trace", "span"));+ for key in ["input", "output", "usage_details", "completion_start_time"] {+ assert!(!attrs.contains_key(&format!("langfuse.observation.{key}")));+ }+ assert_eq!(attrs["langfuse.observation.model.name"], "requested-model");+ assert!(metadata(&attrs).get("native_usage").is_none());+ }++ #[test]+ fn usage_projection_preserves_new_numeric_details_without_unknown_top_level_fields() {+ let mut facts = facts();+ let usage = json!({+ "input_tokens": 20, "output_tokens": 10, "total_tokens": 30,+ "input_tokens_details": {"cached_tokens": 4, "future_counter": 3, "optional": null, "future_object": {}},+ "output_tokens_details": null,+ "future_usage": {"cost": 123},+ });+ facts.inference.usage_details = Some(usage.clone());+ let attrs = attributes(&span(facts, "trace", "span"));+ assert_eq!(metadata(&attrs)["native_usage"], usage);+ let projected: Value = serde_json::from_str(+ attrs["langfuse.observation.usage_details"]+ .as_str()+ .unwrap(),+ )+ .unwrap();+ assert_eq!(+ projected,+ json!({+ "input_tokens": 20, "output_tokens": 10, "total_tokens": 30,+ "input_tokens_details": {"cached_tokens": 4, "future_counter": 3, "optional": null},+ "output_tokens_details": null,+ })+ );+ }++ #[test]+ fn unsupported_usage_remains_metadata_without_invented_totals() {+ let mut facts = facts();+ facts.inference.usage_details = Some(json!({"input_tokens": 9}));+ let attrs = attributes(&span(facts, "trace", "span"));+ assert!(!attrs.contains_key("langfuse.observation.usage_details"));+ assert_eq!(metadata(&attrs)["native_usage"], json!({"input_tokens": 9}));+ }++ #[test]+ fn provider_and_transport_failures_are_errors_while_cancellation_stays_distinct() {+ for (http_status, provider_status, outcome, level, code) in [+ (Some(429), None, RelayOutcome::Eof, "ERROR", 2),+ (Some(200), Some("failed"), RelayOutcome::Eof, "ERROR", 2),+ (None, None, RelayOutcome::TransportError, "ERROR", 2),+ (Some(200), None, RelayOutcome::Timeout, "ERROR", 2),+ (Some(200), None, RelayOutcome::Cancelled, "WARNING", 0),+ (+ Some(200),+ Some("incomplete"),+ RelayOutcome::Eof,+ "WARNING",+ 0,+ ),+ ] {+ let mut facts = facts();+ facts.http_status = http_status;+ facts.inference.provider_status = provider_status.map(str::to_owned);+ facts.outcome = outcome;+ let span = span(facts, "trace", "span");+ let attrs = attributes(&span);+ assert_eq!(attrs["langfuse.observation.level"], level);+ assert_eq!(span["status"]["code"], code);+ assert_eq!(metadata(&attrs)["relay_outcome"], json!(outcome));+ }+ }+}diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rsnew file mode 100644index 0000000000..7a9b2383a3--- /dev/null+++ b/ai-gateway/src/telemetry/mod.rs@@ -0,0 +1,208 @@+//! Immediate, best-effort delivery of finalized inference facts.+mod mapping;+mod otlp;++use std::{+ io::{self, Write},+ sync::{+ Arc, Mutex,+ atomic::{AtomicU64, Ordering},+ },+};++use tokio::{sync::Semaphore, task::JoinSet, time::Instant};+use uuid::Uuid;++use crate::{+ capture::InferenceFacts,+ resolution::{ControlPlaneConfig, ResolutionError, ResolvedRequestContext},+};+use otlp::Uploader;++const MAX_UPLOADS: usize = 32;+const MAX_FACT_BYTES: usize = 4 * 1024 * 1024;+const MAX_RETAINED_BYTES: usize = 16 * 1024 * 1024;++/// Credentials are deliberately not serializable or printable. Project attribution+/// stays attached to each execution even when delivery outlives the response body.+pub(crate) struct DeliveryContext {+ pub project_id: String,+ pub access_token: String,+ pub expires_at: u64,+ trace_id: String,+ observation_id: String,+}++impl DeliveryContext {+ pub fn from_resolved(context: &ResolvedRequestContext) -> Self {+ Self {+ project_id: context.attribution().project_id().to_owned(),+ access_token: context.ingestion().access_token().to_owned(),+ expires_at: context.ingestion().expires_at(),+ trace_id: Uuid::new_v4().simple().to_string(),+ observation_id: Uuid::new_v4().simple().to_string()[..16].to_owned(),+ }+ }+}++#[derive(Default)]+struct Stats {+ accepted: AtomicU64,+ failed: AtomicU64,+ dropped: AtomicU64,+}++struct Delivery {+ uploader: Arc<Uploader>,+ capacity: Arc<Semaphore>,+ bytes: Arc<Semaphore>,+ tasks: Mutex<Option<JoinSet<()>>>,+ stats: Arc<Stats>,+}++/// Shared delivery handle. Shutdown closes submission and drains already admitted work.+#[derive(Clone)]+pub struct Telemetry(Arc<Delivery>);++impl Telemetry {+ pub(crate) fn new(config: &ControlPlaneConfig) -> Result<Self, ResolutionError> {+ Ok(Self::with_uploader(+ Uploader::new(config)?,+ MAX_UPLOADS,+ MAX_RETAINED_BYTES,+ ))+ }++ fn with_uploader(uploader: Uploader, uploads: usize, bytes: usize) -> Self {+ Self(Arc::new(Delivery {+ uploader: Arc::new(uploader),+ capacity: Arc::new(Semaphore::new(uploads)),+ bytes: Arc::new(Semaphore::new(bytes)),+ tasks: Mutex::new(Some(JoinSet::new())),+ stats: Arc::new(Stats::default()),+ }))+ }++ /// No waiting queue and no network work on the caller's response/drop path.+ pub(crate) fn record(&self, context: DeliveryContext, facts: InferenceFacts) {+ let Ok(capacity) = self.0.capacity.clone().try_acquire_owned() else {+ self.drop_record("capacity");+ return;+ };+ let mut size = SizeCounter {+ bytes: context.access_token.len() + context.project_id.len(),+ limit: MAX_FACT_BYTES,+ };+ if serde_json::to_writer(&mut size, &facts).is_err() {+ self.drop_record("size");+ return;+ }+ let Ok(bytes) = self+ .0+ .bytes+ .clone()+ .try_acquire_many_owned(u32::try_from(size.bytes).expect("bounded fact bytes"))+ else {+ self.drop_record("bytes");+ return;+ };+ let mut tasks = self.0.tasks.lock().expect("telemetry task lock poisoned");+ let Some(tasks) = tasks.as_mut() else {+ self.drop_record("shutdown");+ return;+ };+ // Reap completed handles on submission so the task registry stays bounded.+ while tasks.try_join_next().is_some() {}+ let uploader = self.0.uploader.clone();+ let stats = self.0.stats.clone();+ tasks.spawn(async move {+ let _capacity = capacity;+ let _bytes = bytes;+ let span = mapping::span(facts, &context.trace_id, &context.observation_id);+ match uploader.export(&context, &[span]).await {+ Ok(()) => {+ stats.accepted.fetch_add(1, Ordering::Relaxed);+ tracing::debug!("gateway telemetry accepted");+ }+ Err(error) => {+ stats.failed.fetch_add(1, Ordering::Relaxed);+ // Error categories contain no URLs, credentials, response bodies or content.+ tracing::warn!(reason = error.reason(), "gateway telemetry upload failed");+ }+ }+ });+ }++ fn drop_record(&self, reason: &'static str) {+ self.0.stats.dropped.fetch_add(1, Ordering::Relaxed);+ tracing::warn!(reason, "gateway telemetry dropped");+ }++ /// Stop accepting work and finish uploads within the existing process drain deadline.+ ///+ /// # Panics+ /// Panics if the upload task lock was poisoned.+ pub async fn shutdown(&self, deadline: Instant) {+ self.0.capacity.close();+ let tasks = self+ .0+ .tasks+ .lock()+ .expect("telemetry task lock poisoned")+ .take();+ let Some(mut tasks) = tasks else {+ return;+ };+ loop {+ match tokio::time::timeout_at(deadline, tasks.join_next()).await {+ Ok(Some(Ok(()))) => {}+ Ok(Some(Err(_))) => {+ self.0.stats.failed.fetch_add(1, Ordering::Relaxed);+ }+ Ok(None) => break,+ Err(_) => {+ tasks.abort_all();+ while let Some(result) = tasks.join_next().await {+ if result.is_err() {+ self.0.stats.dropped.fetch_add(1, Ordering::Relaxed);+ }+ }+ break;+ }+ }+ }+ tracing::info!(+ accepted = self.0.stats.accepted.load(Ordering::Relaxed),+ failed = self.0.stats.failed.load(Ordering::Relaxed),+ dropped = self.0.stats.dropped.load(Ordering::Relaxed),+ "gateway telemetry stopped"+ );+ }+}++/// Count serialized facts without allocating a second copy on the finalization path.+struct SizeCounter {+ bytes: usize,+ limit: usize,+}++impl Write for SizeCounter {+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {+ if bytes.len() > self.limit.saturating_sub(self.bytes) {+ return Err(io::Error::other("telemetry size limit"));+ }+ self.bytes += bytes.len();+ Ok(bytes.len())+ }+ fn flush(&mut self) -> io::Result<()> {+ Ok(())+ }+}++pub(crate) fn debug_record(facts: &InferenceFacts) {+ // Full mode intentionally includes native request and completed output content.+ tracing::debug!(capture = %format_args!("{:#}", serde_json::to_value(facts).expect("captured facts must serialize")), "gateway response captured");+}++#[cfg(test)]+mod tests;diff --git a/ai-gateway/src/telemetry/otlp.rs b/ai-gateway/src/telemetry/otlp.rsnew file mode 100644index 0000000000..c35a5d1528--- /dev/null+++ b/ai-gateway/src/telemetry/otlp.rs@@ -0,0 +1,165 @@+use std::{+ io::{self, Write},+ time::{Duration, SystemTime, UNIX_EPOCH},+};++use axum::http::{HeaderValue, header};+use reqwest::{Client, Url};+use serde_json::{Value, json};++use super::DeliveryContext;+use crate::resolution::{ControlPlaneConfig, ResolutionError, signing};++const INGESTION_PATH: &str = "/api/public/otel/v1/traces";+const MAX_PAYLOAD_BYTES: usize = 8 * 1024 * 1024;+const MAX_RESPONSE_BYTES: usize = 64 * 1024;+const UPLOAD_TIMEOUT: Duration = Duration::from_secs(5);++pub(super) struct Uploader {+ client: Client,+ endpoint: Url,+ service_key: String,+}++#[derive(Debug, PartialEq, Eq)]+pub(super) enum ExportError {+ Expired,+ Payload,+ Transport,+ Rejected,+ Response,+ Partial,+}++impl ExportError {+ pub fn reason(&self) -> &'static str {+ match self {+ Self::Expired => "expired_grant",+ Self::Payload => "payload",+ Self::Transport => "transport",+ Self::Rejected => "http_status",+ Self::Response => "invalid_response",+ Self::Partial => "rejected_spans",+ }+ }+}++impl Uploader {+ pub fn new(config: &ControlPlaneConfig) -> Result<Self, ResolutionError> {+ let client = Client::builder()+ .redirect(reqwest::redirect::Policy::none())+ .retry(reqwest::retry::never())+ .no_proxy()+ .no_gzip()+ .no_brotli()+ .no_deflate()+ .no_zstd()+ .connect_timeout(Duration::from_secs(2))+ .timeout(UPLOAD_TIMEOUT)+ .pool_max_idle_per_host(16)+ .build()+ .map_err(|_| ResolutionError::Configuration)?;+ Ok(Self {+ client,+ endpoint: config.endpoint(INGESTION_PATH),+ service_key: config.service_key().to_owned(),+ })+ }++ /// A collection allows project batching without changing the HTTP transport.+ pub async fn export(+ &self,+ grant: &DeliveryContext,+ spans: &[Value],+ ) -> Result<(), ExportError> {+ let payload = json!({"resourceSpans": [{+ "resource": {"attributes": [{"key": "service.name", "value": {"stringValue": "langfuse-ai-gateway"}}]},+ "scopeSpans": [{"scope": {"name": "langfuse-ai-gateway", "version": env!("CARGO_PKG_VERSION")}, "spans": spans}]+ }]});+ let mut body = LimitedBody(Vec::new());+ serde_json::to_writer(&mut body, &payload).map_err(|_| ExportError::Payload)?;+ let now = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map_err(|_| ExportError::Expired)?;+ let remaining = Duration::from_secs(grant.expires_at)+ .checked_sub(now)+ .filter(|duration| !duration.is_zero())+ .ok_or(ExportError::Expired)?;+ let mut authorization = HeaderValue::from_str(&format!("Bearer {}", grant.access_token))+ .map_err(|_| ExportError::Payload)?;+ authorization.set_sensitive(true);+ let mut signature = HeaderValue::from_str(&signing::authorization(+ &self.service_key,+ &grant.access_token,+ now.as_secs(),+ ))+ .map_err(|_| ExportError::Payload)?;+ signature.set_sensitive(true);+ let mut response = self+ .client+ .post(self.endpoint.clone())+ .header(header::AUTHORIZATION, authorization)+ .header("langfuse-gateway-authorization", signature)+ .header(header::CONTENT_TYPE, "application/json")+ .header(header::ACCEPT_ENCODING, "identity")+ .header("x-langfuse-sdk-name", "langfuse-ai-gateway")+ .header("x-langfuse-sdk-version", env!("CARGO_PKG_VERSION"))+ .header("x-langfuse-ingestion-version", "4")+ .timeout(UPLOAD_TIMEOUT.min(remaining))+ .body(body.0)+ .send()+ .await+ .map_err(|_| ExportError::Transport)?;+ if response.status().as_u16() != 200 {+ return Err(ExportError::Rejected);+ }+ if response+ .content_length()+ .is_some_and(|size| size > MAX_RESPONSE_BYTES as u64)+ {+ return Err(ExportError::Response);+ }+ let mut bytes = Vec::new();+ while let Some(chunk) = response.chunk().await.map_err(|_| ExportError::Transport)? {+ if chunk.len() > MAX_RESPONSE_BYTES - bytes.len() {+ return Err(ExportError::Response);+ }+ bytes.extend_from_slice(&chunk);+ }+ let result: Value = serde_json::from_slice(&bytes).map_err(|_| ExportError::Response)?;+ if !result.is_object() {+ return Err(ExportError::Response);+ }+ if let Some(partial) = result.get("partialSuccess") {+ if !partial.is_object() {+ return Err(ExportError::Response);+ }+ let rejected = match partial.get("rejectedSpans") {+ None => 0,+ Some(Value::String(value)) => {+ value.parse::<u64>().map_err(|_| ExportError::Response)?+ }+ Some(value) => value.as_u64().ok_or(ExportError::Response)?,+ };+ if rejected > 0 {+ return Err(ExportError::Partial);+ }+ }+ Ok(())+ }+}++struct LimitedBody(Vec<u8>);++impl Write for LimitedBody {+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {+ if bytes.len() > MAX_PAYLOAD_BYTES - self.0.len() {+ return Err(io::Error::other("telemetry payload limit"));+ }+ self.0.extend_from_slice(bytes);+ Ok(bytes.len())+ }+ fn flush(&mut self) -> io::Result<()> {+ Ok(())+ }+}diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rsnew file mode 100644index 0000000000..2b1bd50c3a--- /dev/null+++ b/ai-gateway/src/telemetry/tests.rs@@ -0,0 +1,320 @@+use std::{sync::Arc, time::Duration};++use axum::{+ body::{Body, to_bytes},+ http::Response,+};+use serde_json::{Value, json};+use tokio::sync::Notify;++use super::{otlp::ExportError, *};+use crate::{+ capture::RelayOutcome,+ resolution::signing,+ test_support::{FakeServer, resolved_request_context},+};++fn facts(project: &str) -> InferenceFacts {+ InferenceFacts {+ api_format: "openai.responses",+ start_time_unix_ms: 1_735_689_600_000,+ duration_ms: 100,+ first_byte_ms: Some(10),+ http_status: Some(200),+ metadata: json!({"project_id": project, "ingestion_mode": "usage"}),+ outcome: RelayOutcome::Eof,+ inference: crate::capture::ProviderFacts::default(),+ }+}++async fn grant() -> DeliveryContext {+ DeliveryContext::from_resolved(&resolved_request_context("provider-secret").await)+}++fn uploader(url: &str) -> Uploader {+ Uploader::new(&ControlPlaneConfig::new(url, "test-service-key").unwrap()).unwrap()+}++fn response(status: u16, body: impl Into<Body>) -> Response<Body> {+ Response::builder()+ .status(status)+ .body(body.into())+ .unwrap()+}++#[tokio::test]+async fn immediate_upload_uses_prefixed_path_signed_grant_and_gateway_sdk_headers() {+ let web = FakeServer::start(|request| async move {+ assert_eq!(request.method(), "POST");+ assert_eq!(request.uri().path(), "/app/api/public/otel/v1/traces");+ let headers = request.headers();+ assert_eq!(headers["authorization"], "Bearer private-ingestion-token");+ let signature = headers["langfuse-gateway-authorization"].to_str().unwrap();+ let timestamp = signature+ .strip_prefix("HMAC timestamp=")+ .unwrap()+ .split_once(',')+ .unwrap()+ .0+ .parse()+ .unwrap();+ assert_eq!(+ signature,+ signing::authorization("test-service-key", "private-ingestion-token", timestamp)+ );+ assert_eq!(headers["content-type"], "application/json");+ assert_eq!(headers["x-langfuse-sdk-name"], "langfuse-ai-gateway");+ assert_eq!(headers["x-langfuse-sdk-version"], env!("CARGO_PKG_VERSION"));+ assert_eq!(headers["x-langfuse-ingestion-version"], "4");+ let bytes = to_bytes(request.into_body(), 64 * 1024).await.unwrap();+ let payload: Value = serde_json::from_slice(&bytes).unwrap();+ let spans = payload["resourceSpans"][0]["scopeSpans"][0]["spans"]+ .as_array()+ .unwrap();+ assert_eq!(spans.len(), 1);+ assert_eq!(spans[0]["traceId"].as_str().unwrap().len(), 32);+ assert_eq!(spans[0]["spanId"].as_str().unwrap().len(), 16);+ let body = String::from_utf8(bytes.to_vec()).unwrap();+ assert!(!body.contains("provider-secret"));+ assert!(!body.contains("private-ingestion-token"));+ response(200, "{}")+ })+ .await;+ let telemetry = Telemetry::new(+ &ControlPlaneConfig::new(&format!("{}/app", web.url), "test-service-key").unwrap(),+ )+ .unwrap();+ telemetry.record(grant().await, facts("project-1"));+ telemetry+ .shutdown(Instant::now() + Duration::from_secs(2))+ .await;+ assert_eq!(web.calls(), 1);+ assert_eq!(telemetry.0.stats.accepted.load(Ordering::Relaxed), 1);+ assert_eq!(telemetry.0.stats.failed.load(Ordering::Relaxed), 0);+}++#[tokio::test]+async fn concurrent_projects_keep_their_original_grants_and_attribution() {+ let observed = Arc::new(Mutex::new(Vec::new()));+ let received = observed.clone();+ let both_uploads = Arc::new(tokio::sync::Barrier::new(2));+ let web = FakeServer::start(move |request| {+ let received = received.clone();+ let both_uploads = both_uploads.clone();+ async move {+ both_uploads.wait().await;+ let authorization = request.headers()["authorization"]+ .to_str()+ .unwrap()+ .to_owned();+ let bytes = to_bytes(request.into_body(), 64 * 1024).await.unwrap();+ let payload: Value = serde_json::from_slice(&bytes).unwrap();+ let attributes = payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]+ .as_array()+ .unwrap();+ let metadata = attributes+ .iter()+ .find(|attribute| attribute["key"] == "langfuse.observation.metadata")+ .unwrap();+ let metadata: Value =+ serde_json::from_str(metadata["value"]["stringValue"].as_str().unwrap()).unwrap();+ received.lock().unwrap().push((+ authorization,+ metadata["project_id"].as_str().unwrap().to_owned(),+ ));+ response(200, "{}")+ }+ })+ .await;+ let telemetry = Telemetry::with_uploader(uploader(&web.url), 2, MAX_RETAINED_BYTES);+ for project in ["project-a", "project-b"] {+ let mut grant = grant().await;+ grant.project_id = project.into();+ grant.access_token = format!("token-{project}");+ telemetry.record(grant, facts(project));+ }+ telemetry+ .shutdown(Instant::now() + Duration::from_secs(2))+ .await;+ let mut actual = observed.lock().unwrap().clone();+ actual.sort();+ assert_eq!(+ actual,+ vec![+ ("Bearer token-project-a".into(), "project-a".into()),+ ("Bearer token-project-b".into(), "project-b".into())+ ]+ );+ assert_eq!(telemetry.0.stats.accepted.load(Ordering::Relaxed), 2);+}++#[tokio::test]+async fn upload_response_failures_are_reported_without_retries() {+ let grant = grant().await;+ for (status, body, expected) in [+ (500, "{}", Err(ExportError::Rejected)),+ (+ 200,+ r#"{"partialSuccess":{"rejectedSpans":"1"}}"#,+ Err(ExportError::Partial),+ ),+ (+ 200,+ r#"{"partialSuccess":{"rejectedSpans":1}}"#,+ Err(ExportError::Partial),+ ),+ (+ 200,+ r#"{"partialSuccess":{"rejectedSpans":"bad"}}"#,+ Err(ExportError::Response),+ ),+ (200, "[]", Err(ExportError::Response)),+ (200, "not json", Err(ExportError::Response)),+ (200, r#"{"partialSuccess":{"rejectedSpans":"0"}}"#, Ok(())),+ ] {+ let web = FakeServer::start(move |_| async move { response(status, body) }).await;+ assert_eq!(+ uploader(&web.url).export(&grant, &[json!({})]).await,+ expected+ );+ assert_eq!(web.calls(), 1);+ }+ // Streaming without Content-Length must obey the response bound too.+ let web = FakeServer::start(|_| async move {+ let chunks = futures_util::stream::iter([Ok::<_, std::io::Error>(vec![b' '; 65 * 1024])]);+ response(200, Body::from_stream(chunks))+ })+ .await;+ assert_eq!(+ uploader(&web.url).export(&grant, &[json!({})]).await,+ Err(ExportError::Response)+ );+ assert_eq!(web.calls(), 1);+}++#[tokio::test]+async fn redirects_do_not_forward_the_ingestion_credentials() {+ let destination = FakeServer::start(|_| async { response(200, "{}") }).await;+ let location = destination.url.clone();+ let web = FakeServer::start(move |_| {+ let location = location.clone();+ async move {+ Response::builder()+ .status(307)+ .header("location", location)+ .body(Body::empty())+ .unwrap()+ }+ })+ .await;+ assert_eq!(+ uploader(&web.url)+ .export(&grant().await, &[json!({})])+ .await,+ Err(ExportError::Rejected)+ );+ assert_eq!(web.calls(), 1);+ assert_eq!(destination.calls(), 0);+}++#[tokio::test]+async fn expired_grants_and_oversized_payloads_never_reach_the_network() {+ let web = FakeServer::start(|_| async { response(200, "{}") }).await;+ let uploader = uploader(&web.url);+ let mut expired = grant().await;+ expired.expires_at = 1;+ assert_eq!(+ uploader.export(&expired, &[json!({})]).await,+ Err(ExportError::Expired)+ );+ let oversized = json!({"oversized": "x".repeat(8 * 1024 * 1024)});+ assert_eq!(+ uploader.export(&grant().await, &[oversized]).await,+ Err(ExportError::Payload)+ );+ assert_eq!(web.calls(), 0);+}++#[tokio::test]+async fn upload_capacity_and_shutdown_deadline_bound_hanging_delivery() {+ let started = Arc::new(Notify::new());+ let received = started.clone();+ let web = FakeServer::start(move |_| {+ let received = received.clone();+ async move {+ received.notify_one();+ std::future::pending::<()>().await;+ response(200, "{}")+ }+ })+ .await;+ let telemetry = Telemetry::with_uploader(uploader(&web.url), 1, MAX_RETAINED_BYTES);+ telemetry.record(grant().await, facts("project-1"));+ tokio::time::timeout(Duration::from_secs(2), started.notified())+ .await+ .unwrap();+ telemetry.record(grant().await, facts("project-1"));+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 1);+ telemetry.shutdown(Instant::now()).await;+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 2);+ assert_eq!(telemetry.0.capacity.available_permits(), 1);+ assert_eq!(telemetry.0.bytes.available_permits(), MAX_RETAINED_BYTES);+ telemetry.record(grant().await, facts("project-1"));+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 3);+ assert_eq!(web.calls(), 1);+}++#[tokio::test]+async fn retained_byte_budget_is_released_after_successful_drain() {+ let started = Arc::new(Notify::new());+ let release = Arc::new(Notify::new());+ let received = started.clone();+ let gate = release.clone();+ let web = FakeServer::start(move |_| {+ let received = received.clone();+ let gate = gate.clone();+ async move {+ received.notify_one();+ gate.notified().await;+ response(200, "{}")+ }+ })+ .await;+ let grant = grant().await;+ let budget = serde_json::to_vec(&facts("project-1")).unwrap().len()+ + grant.access_token.len()+ + grant.project_id.len();+ let telemetry = Telemetry::with_uploader(uploader(&web.url), 2, budget);+ telemetry.record(grant, facts("project-1"));+ tokio::time::timeout(Duration::from_secs(2), started.notified())+ .await+ .unwrap();+ assert_eq!(telemetry.0.bytes.available_permits(), 0);+ let second = DeliveryContext::from_resolved(&resolved_request_context("provider-secret").await);+ telemetry.record(second, facts("project-1"));+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 1);+ release.notify_one();+ telemetry+ .shutdown(Instant::now() + Duration::from_secs(2))+ .await;+ assert_eq!(telemetry.0.bytes.available_permits(), budget);+ assert_eq!(telemetry.0.capacity.available_permits(), 2);+ assert_eq!(telemetry.0.stats.accepted.load(Ordering::Relaxed), 1);+ assert_eq!(web.calls(), 1);+}++#[tokio::test]+async fn oversized_facts_are_dropped_before_task_admission() {+ let web = FakeServer::start(|_| async { response(200, "{}") }).await;+ let telemetry = Telemetry::with_uploader(uploader(&web.url), 1, MAX_RETAINED_BYTES);+ let mut oversized = facts("project-1");+ oversized.metadata["oversized"] = json!("x".repeat(MAX_FACT_BYTES));+ telemetry.record(grant().await, oversized);+ assert_eq!(telemetry.0.stats.dropped.load(Ordering::Relaxed), 1);+ assert_eq!(telemetry.0.capacity.available_permits(), 1);+ assert_eq!(telemetry.0.bytes.available_permits(), MAX_RETAINED_BYTES);+ telemetry+ .shutdown(Instant::now() + Duration::from_secs(2))+ .await;+ assert_eq!(web.calls(), 0);+}diff --git a/packages/shared/scripts/seeder/seed-postgres.ts b/packages/shared/scripts/seeder/seed-postgres.tsindex d9a78bb796..6e7f61312b 100644--- a/packages/shared/scripts/seeder/seed-postgres.ts+++ b/packages/shared/scripts/seeder/seed-postgres.ts@@ -557,7 +557,7 @@ async function seedAiGateway(params: {create: {organizationId: params.organizationId,defaultIngestionProjectId: ingestionProjectId,- ingestionMode: "USAGE",+ ingestionMode: "FULL",},update: {defaultIngestionProjectId: ingestionProjectId,diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.tsindex fb511e8f8a..549dee4acd 100644--- a/worker/src/services/ClickhouseWriter/index.ts+++ b/worker/src/services/ClickhouseWriter/index.ts@@ -90,8 +90,6 @@ export class ClickhouseWriter {this.isIntervalFlushInProgress = true;- logger.debug("Flush interval elapsed, flushing all queues...");-this.flushAll().finally(() => {this.isIntervalFlushInProgress = false;});