🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ingestion): attribute AI gateway spans to their gateway API key and service (#19398)
Tobias Wochingera4467fd41800
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lockindex f22f5e4cd5..69e33b89ba 100644--- a/ai-gateway/Cargo.lock+++ b/ai-gateway/Cargo.lock@@ -19,7 +19,7 @@ dependencies = [[[package]]name = "ai-gateway"-version = "0.1.0"+version = "1.0.0"dependencies = ["axum","base64 0.23.1",diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.tomlindex c1052881af..9f3cf35ac7 100644--- a/ai-gateway/Cargo.toml+++ b/ai-gateway/Cargo.toml@@ -1,6 +1,6 @@[package]name = "ai-gateway"-version = "0.1.0"+version = "1.0.0"edition = "2024"publish = falselicense = "MIT"diff --git a/ai-gateway/src/telemetry/mapping.rs b/ai-gateway/src/telemetry/mapping.rsindex f87702c7a2..f303f30385 100644--- a/ai-gateway/src/telemetry/mapping.rs+++ b/ai-gateway/src/telemetry/mapping.rs@@ -531,6 +531,39 @@ mod tests {assert!(!attrs.contains_key("langfuse.trace.metadata"));}+ // Langfuse stores this metadata key as the span's ingestion API key for+ // gateway uploads, so callers must never be able to set it.+ #[test]+ fn caller_headers_never_supply_the_gateway_key_id() {+ let mut headers = axum::http::HeaderMap::new();+ headers.insert(+ "langfuse-metadata",+ "langfuse.gateway.api_key.id:header-key".parse().unwrap(),+ );+ headers.insert(+ "baggage",+ "langfuse_metadata_langfuse.gateway.api_key.id=baggage-key"+ .parse()+ .unwrap(),+ );+ let context = GenerationContext {+ trace_id: "0123456789abcdef0123456789abcdef".into(),+ observation_id: "0123456789abcdef".into(),+ ..GenerationContext::from_headers(&headers)+ };+ assert!(context.metadata.contains_key("langfuse.gateway.api_key.id"));++ let mut unattributed = facts();+ unattributed.metadata["key_id"] = Value::Null;+ for (facts, expected) in [(facts(), Some("key")), (unattributed, None)] {+ let metadata = metadata(&attributes(&span(facts, &context)));+ assert_eq!(+ metadata.get("langfuse.gateway.api_key.id"),+ expected.map(Value::from).as_ref()+ );+ }+ }+#[test]fn full_generation_preserves_native_content_attribution_and_completion_time() {let span = span(facts(), &context());diff --git a/ai-gateway/src/telemetry/tests.rs b/ai-gateway/src/telemetry/tests.rsindex 93545c1c79..50d48f36f3 100644--- a/ai-gateway/src/telemetry/tests.rs+++ b/ai-gateway/src/telemetry/tests.rs@@ -97,6 +97,16 @@ async fn upload_uses_prefixed_path_signed_grant_and_gateway_sdk_headers() {assert_eq!(headers["x-langfuse-ingestion-version"], "4");let bytes = to_bytes(request.into_body(), 64 * 1024).await.unwrap();let payload = upload_json(&bytes);+ // No `service.version`: ingestion would use it as the generation's+ // user-facing version. Langfuse takes the gateway version from the scope.+ assert_eq!(+ payload["resourceSpans"][0]["resource"]["attributes"],+ json!([{"key": "service.name", "value": {"stringValue": "langfuse-ai-gateway"}}])+ );+ assert_eq!(+ payload["resourceSpans"][0]["scopeSpans"][0]["scope"]["version"],+ env!("CARGO_PKG_VERSION")+ );let spans = payload["resourceSpans"][0]["scopeSpans"][0]["spans"].as_array().unwrap();diff --git a/packages/shared/src/features/ai-gateway/constants.ts b/packages/shared/src/features/ai-gateway/constants.tsnew file mode 100644index 0000000000..f5c2fb947f--- /dev/null+++ b/packages/shared/src/features/ai-gateway/constants.ts@@ -0,0 +1,5 @@+/**+ * OTel `service.name` of the AI gateway. On the events table, only spans the+ * gateway uploaded carry it as their `service_name`.+ */+export const AI_GATEWAY_SERVICE_NAME = "langfuse-ai-gateway";diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.tsindex fa367b4801..b72f4fdf44 100644--- a/packages/shared/src/index.ts+++ b/packages/shared/src/index.ts@@ -102,6 +102,9 @@ export * from "./features/skills/types";export * from "./features/skills/trace";export * from "./features/skills/constants";+// ai gateway+export * from "./features/ai-gateway/constants";+export {compileChatMessages,compileChatMessagesWithIds,diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex 7ed785b3ea..6c65d80ecb 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -49,6 +49,9 @@ export const AI_GATEWAY_INSTRUMENTATION_SCOPE_NAME = "langfuse-ai-gateway";const AI_GATEWAY_INGESTION_API_KEY_PREFIX = "gateway:";+/** Metadata key under which the AI gateway records the caller's gateway API key id. */+export const AI_GATEWAY_KEY_ID_METADATA_KEY = "langfuse.gateway.api_key.id";+/*** The ingestion API key Web records for an upload authorized by a gateway* ingestion grant. Unlike the instrumentation scope name, an OTLP sender cannotdiff --git a/web/src/__e2e__/ai-gateway.gatewaye2e.ts b/web/src/__e2e__/ai-gateway.gatewaye2e.tsindex 33395c106b..7a408cef73 100644--- a/web/src/__e2e__/ai-gateway.gatewaye2e.ts+++ b/web/src/__e2e__/ai-gateway.gatewaye2e.ts@@ -886,6 +886,7 @@ async function ingestTrace(// The payload claims another project everywhere a span can; ingestion must still// write only into the project named by the gateway token.const spoofedProjectId = `spoofed-${randomBytes(8).toString("hex")}`;+ const gatewayKeyId = `gateway-key-${randomBytes(8).toString("hex")}`;const spoofedProject = {key: "langfuse.project.id",value: { stringValue: spoofedProjectId },@@ -923,6 +924,7 @@ async function ingestTrace(value: {stringValue: JSON.stringify({"langfuse.gateway.project.id": spoofedProjectId,+ "langfuse.gateway.api_key.id": gatewayKeyId,}),},},@@ -966,6 +968,20 @@ async function ingestTrace(expect(trace?.id).toBe(traceId);expect(observation?.id).toBe(observationId);await expectStoredInOneProject({ observationId, traceId });+ const attribution = await queryClickhouse<{+ ingestion_api_key: string;+ service_name: string;+ }>({+ query: `SELECT ingestion_api_key, service_name FROM events_core+ WHERE span_id = {observationId: String} AND trace_id = {traceId: String}`,+ params: { observationId, traceId },+ });+ expect(attribution).toEqual([+ {+ ingestion_api_key: gatewayKeyId,+ service_name: "langfuse-ai-gateway",+ },+ ]);},40_000,1_000,diff --git a/worker/src/services/IngestionService/aiGatewayAttribution.ts b/worker/src/services/IngestionService/aiGatewayAttribution.tsnew file mode 100644index 0000000000..fca4297600--- /dev/null+++ b/worker/src/services/IngestionService/aiGatewayAttribution.ts@@ -0,0 +1,50 @@+import { AI_GATEWAY_SERVICE_NAME } from "@langfuse/shared";+import {+ AI_GATEWAY_KEY_ID_METADATA_KEY,+ isAiGatewayIngestionApiKey,+} from "@langfuse/shared/src/server";++/**+ * Key and service a span is attributed to on the events table.+ *+ * An upload the AI gateway authenticated (ingestion key `gateway:<grant>`)+ * batches requests made with different gateway API keys. Each span names its+ * key in `langfuse.gateway.api_key.id`, which the gateway always overwrites, so+ * that key id becomes the span's ingestion API key. The service is the gateway,+ * versioned by its instrumentation scope: the gateway sends no `service.version`+ * because ingestion would use it as the generation's version. A gateway span+ * without a key id is rejected.+ */+export const resolveAiGatewayAttribution = (params: {+ ingestionApiKey: string | null | undefined;+ metadata: Record<string, unknown>;+ serviceName: string | null | undefined;+ serviceVersion: string | null | undefined;+ scopeVersion: string | null | undefined;+}): {+ ingestionApiKey: string;+ serviceName: string | null | undefined;+ serviceVersion: string | null | undefined;+} => {+ if (!isAiGatewayIngestionApiKey(params.ingestionApiKey)) {+ // Real gateway spans never get here, but an SDK sending straight to+ // Langfuse can claim the gateway's service name; such spans lose it.+ const spoofsGateway = params.serviceName === AI_GATEWAY_SERVICE_NAME;+ return {+ ingestionApiKey: params.ingestionApiKey ?? "",+ serviceName: spoofsGateway ? undefined : params.serviceName,+ serviceVersion: spoofsGateway ? undefined : params.serviceVersion,+ };+ }+ const keyId = params.metadata[AI_GATEWAY_KEY_ID_METADATA_KEY];+ if (typeof keyId !== "string" || keyId.length === 0) {+ throw new Error(+ `AI gateway span is missing ${AI_GATEWAY_KEY_ID_METADATA_KEY} in its metadata`,+ );+ }+ return {+ ingestionApiKey: keyId,+ serviceName: AI_GATEWAY_SERVICE_NAME,+ serviceVersion: params.scopeVersion,+ };+};diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex c1e89cadf1..26eb5eadf6 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -78,6 +78,7 @@ import {isAiGatewayGenerationEvent,overwriteObject,} from "./utils";+import { resolveAiGatewayAttribution } from "./aiGatewayAttribution";import { randomUUID } from "crypto";import { SpanKind } from "@opentelemetry/api";import { ClickhouseReadSkipCache } from "../../utils/clickhouseReadSkipCache";@@ -329,6 +330,13 @@ export class IngestionService {eventData.modelParameters,);const metadata = eventData.metadata ?? {};+ const spanAttribution = resolveAiGatewayAttribution({+ ingestionApiKey: eventData.ingestionApiKey,+ metadata,+ serviceName: eventData.serviceName,+ serviceVersion: eventData.serviceVersion,+ scopeVersion: eventData.scopeVersion,+ });// Runs outside the modelName gate below so model-less events with provided// usage are still checked.@@ -483,11 +491,11 @@ export class IngestionService {// Source/instrumentation metadatasource: eventData.source,- ingestion_api_key: eventData.ingestionApiKey ?? "",+ ingestion_api_key: spanAttribution.ingestionApiKey,ingestion_sdk_name: eventData.ingestionSdkName ?? "",ingestion_sdk_version: eventData.ingestionSdkVersion ?? "",- service_name: eventData.serviceName,- service_version: eventData.serviceVersion,+ service_name: spanAttribution.serviceName,+ service_version: spanAttribution.serviceVersion,scope_name: eventData.scopeName,scope_version: eventData.scopeVersion,telemetry_sdk_language: eventData.telemetrySdkLanguage,diff --git a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts b/worker/src/services/IngestionService/tests/IngestionService.unit.test.tsindex 081a593a33..5deb632192 100644--- a/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts+++ b/worker/src/services/IngestionService/tests/IngestionService.unit.test.ts@@ -92,6 +92,78 @@ describe("IngestionService unit tests", () => {expect(acceptedBytes).toBe(eventBytes);});+ describe("AI gateway attribution", () => {+ const createGatewayLikeEventRecord = (overrides: {+ ingestionApiKey: string;+ metadata?: Record<string, unknown>;+ }) =>+ new IngestionService(+ {} as any,+ {} as any,+ { addToQueue: vi.fn() } as any,+ {} as any,+ ).createEventRecord(+ {+ projectId: "project-id",+ traceId: "trace-id",+ spanId: "observation-id",+ parentSpanId: "",+ name: "gateway-generation",+ type: "SPAN",+ environment: "default",+ startTimeISO: "2026-07-22T00:00:00.000Z",+ endTimeISO: "2026-07-22T00:00:01.000Z",+ metadata: overrides.metadata ?? {+ "langfuse.gateway.api_key.id": "gateway-key-1",+ },+ serviceName: "langfuse-ai-gateway",+ serviceVersion: "1.2.3",+ scopeName: "langfuse-ai-gateway",+ scopeVersion: "0.1.0",+ ingestionApiKey: overrides.ingestionApiKey,+ source: "otel",+ },+ "raw-event.json",+ );++ it("attributes a gateway-token upload to the gateway key and its scope-versioned service", async () => {+ const eventRecord = await createGatewayLikeEventRecord({+ ingestionApiKey: "gateway:grant-1",+ });++ expect(eventRecord).toMatchObject({+ ingestion_api_key: "gateway-key-1",+ service_name: "langfuse-ai-gateway",+ service_version: "0.1.0",+ });+ });++ it.each([+ { case: "an SDK key", ingestionApiKey: "pk-lf-spoofing-sdk" },+ { case: "a replay without an ingestion key", ingestionApiKey: "" },+ ])(+ "clears the gateway service claimed by $case and keeps its key",+ async ({ ingestionApiKey }) => {+ const eventRecord = await createGatewayLikeEventRecord({+ ingestionApiKey,+ });++ expect(eventRecord.ingestion_api_key).toBe(ingestionApiKey);+ expect(eventRecord.service_name).toBeUndefined();+ expect(eventRecord.service_version).toBeUndefined();+ },+ );++ it("rejects a gateway-token upload without a gateway key id", async () => {+ await expect(+ createGatewayLikeEventRecord({+ ingestionApiKey: "gateway:grant-1",+ metadata: {},+ }),+ ).rejects.toThrow("langfuse.gateway.api_key.id");+ });+ });+it("overflows only the direct events_full copy and preserves the enriched record", async () => {const addToQueue = vi.fn();const ingestionService = new IngestionService(diff --git a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.tsindex 12e3a74c69..13529eba9f 100644--- a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts+++ b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts@@ -1450,7 +1450,10 @@ describe("Token Cost Calculation", () => {level: "WARNING",statusMessage: "Client cancelled the response",scopeName: gatewayScope.name,- metadata: { scope: gatewayScope },+ metadata: {+ scope: gatewayScope,+ "langfuse.gateway.api_key.id": "gateway-key-1",+ },ingestionApiKey: gatewayAttribution.ingestionApiKey,input: "hello world",output: "[]",@@ -1500,7 +1503,10 @@ describe("Token Cost Calculation", () => {modelName,level: "WARNING",scopeName: gatewayScope.name,- metadata: { scope: gatewayScope },+ metadata: {+ scope: gatewayScope,+ "langfuse.gateway.api_key.id": "gateway-key-1",+ },ingestionApiKey: gatewayAttribution.ingestionApiKey,providedUsageDetails: { input: 10, output: 5 },input: "hello world",