🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ingestion): decode per-key OTel metadata like a blob for next SDK majors (#18436)
Niklas Semmler5979790e2c4a
diff --git a/packages/shared/src/server/ingestion/ingestionAttribution.ts b/packages/shared/src/server/ingestion/ingestionAttribution.tsindex ef03872038..269a1d452c 100644--- a/packages/shared/src/server/ingestion/ingestionAttribution.ts+++ b/packages/shared/src/server/ingestion/ingestionAttribution.ts@@ -25,6 +25,13 @@ export const LANGFUSE_SDK_LATEST_MAJOR = {export type IngestionSdkCanonicalName = keyof typeof LANGFUSE_SDK_LATEST_MAJOR;+// First SDK majors that JSON-encode every per-key metadata value. Ingestion+// decodes those values so they are stored exactly like a metadata blob.+const LANGFUSE_SDK_STRUCTURED_METADATA_MIN_MAJOR = {+ python: 5,+ javascript: 6,+} as const satisfies Record<IngestionSdkCanonicalName, number>;+export type IngestionSdkUpgradeStatus =| "current"| "outdated_major"@@ -134,6 +141,22 @@ export const extractBaseIngestionSdkVersion = (sdkVersion: string): string => {return version;};+export const isStructuredMetadataSdkVersion = (params: {+ sdkName: IngestionSdkCanonicalName | null;+ sdkVersion: string | null | undefined;+}): boolean => {+ if (!params.sdkName || !params.sdkVersion) return false;++ const parsedVersion = parseVersionString(+ extractBaseIngestionSdkVersion(params.sdkVersion),+ );+ return (+ parsedVersion !== null &&+ parsedVersion.major >=+ LANGFUSE_SDK_STRUCTURED_METADATA_MIN_MAJOR[params.sdkName]+ );+};+export const classifyIngestionSdkVersion = (params: {sdkName: string | null | undefined;sdkVersion: string | null | undefined;diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.tsindex 2c82dd4d64..79593183d9 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts@@ -44,6 +44,7 @@ import {type ResourceSpan,} from "./OtelIngestionProcessor";import * as serverBarrel from "../index";+import { flattenJsonToPathArrays } from "./utils";const METRIC = "langfuse.ingestion.metadata_dropped";const ARRAY_ATTRIBUTE_DROPPED_METRIC =@@ -417,6 +418,201 @@ describe("raw OTLP empty attribute values", () => {});});+describe("per-key metadata values from structured-metadata SDK majors", () => {+ const metadata = {+ environment: "prod",+ version: "1.0",+ digits: "42",+ label: "",+ retries: 3,+ ratio: 1.5,+ enabled: true,+ missing: null,+ database: { host: "localhost", port: 5432, auth: { user: "app" } },+ steps: [{ name: "web_fetch", args: { url: "https://example.com" } }],+ empty: {},+ };++ const blobAttributes = (+ domain: "observation" | "trace",+ space?: number,+ ): OtelAttribute[] => [+ {+ key: `langfuse.${domain}.metadata`,+ value: { stringValue: JSON.stringify(metadata, null, space) },+ },+ ];++ // Structured-metadata SDKs JSON-encode every value, strings included.+ const perKeyAttributes = (+ domain: "observation" | "trace",+ space?: number,+ ): OtelAttribute[] =>+ Object.entries(metadata).map(([key, value]) => ({+ key: `langfuse.${domain}.metadata.${key}`,+ value: { stringValue: JSON.stringify(value, null, space) },+ }));++ const processMetadata = async (+ path: "v3" | "v4",+ attributes: OtelAttribute[],+ params: {+ domain?: "observation" | "trace";+ // "child" sends the attributes on a child span of an attribute-less root.+ span?: "root" | "child";+ headerSdkName?: string;+ scopeName?: string;+ scopeVersion?: string;+ telemetrySdkLanguage?: string;+ } = {},+ ) => {+ const scopeVersion = params.scopeVersion ?? "5.0.0";+ const batch = buildBatch(params.span === "child" ? [] : attributes);+ if (params.span === "child") {+ const spans = batch[0].scopeSpans![0].spans!;+ spans.push({+ ...spans[0],+ spanId: Buffer.from("fedcba9876543210", "hex"),+ parentSpanId: spans[0].spanId,+ name: "child-span",+ attributes: [...spans[0].attributes!, ...attributes],+ });+ }+ const scope = batch[0].scopeSpans![0].scope!;+ scope.name = params.scopeName ?? "langfuse-sdk";+ scope.version = scopeVersion;+ if (params.telemetrySdkLanguage) {+ batch[0].resource!.attributes!.push({+ key: "telemetry.sdk.language",+ value: { stringValue: params.telemetrySdkLanguage },+ });+ }+ const processor = new OtelIngestionProcessor({+ projectId: PROJECT_ID,+ publicKey: "pk-test",+ sdkName: params.headerSdkName ?? "python",+ sdkVersion: scopeVersion,+ });+ if (path === "v4") {+ return processor.processToEvent(batch).at(-1)?.metadata as Record<+ string,+ unknown+ >;+ }+ // v3 writes trace metadata to the trace, observation metadata to the span.+ // The last matching event belongs to the span carrying the attributes.+ const eventType =+ params.domain === "trace" ? "trace-create" : "span-create";+ const event = (await processor.processToIngestionEvents(batch))+ .filter((event) => event.type === eventType)+ .at(-1);+ return (event?.body as { metadata?: Record<string, unknown> } | undefined)+ ?.metadata as Record<string, unknown>;+ };++ it.each([+ { path: "v3", domain: "observation", span: "root", encoding: "compact" },+ { path: "v3", domain: "trace", span: "root", encoding: "compact" },+ { path: "v3", domain: "trace", span: "child", encoding: "compact" },+ { path: "v4", domain: "observation", span: "root", encoding: "compact" },+ { path: "v4", domain: "trace", span: "root", encoding: "compact" },+ { path: "v4", domain: "trace", span: "child", encoding: "compact" },+ { path: "v3", domain: "observation", span: "root", encoding: "indented" },+ { path: "v4", domain: "observation", span: "root", encoding: "indented" },+ ] as const)(+ "stores per-key values exactly like a blob ($path, $domain on $span span, $encoding JSON)",+ async ({ path, domain, span, encoding }) => {+ const space = encoding === "indented" ? 2 : undefined;+ const fromBlob = await processMetadata(+ path,+ blobAttributes(domain, space),+ { domain, span },+ );+ const fromPerKey = await processMetadata(+ path,+ perKeyAttributes(domain, space),+ { domain, span },+ );++ expect(fromBlob).toMatchObject(metadata);+ expect(fromPerKey).toEqual(fromBlob);+ expect(flattenJsonToPathArrays(fromPerKey)).toEqual(+ flattenJsonToPathArrays(fromBlob),+ );+ },+ );++ it("keeps a per-key value that is not valid JSON as sent and counts it", async () => {+ recordIncrementMock.mockClear();+ const result = await processMetadata("v4", [+ {+ key: "langfuse.observation.metadata.note",+ value: { stringValue: "{not json" },+ },+ {+ key: "langfuse.observation.metadata.environment",+ value: { stringValue: '"prod"' },+ },+ ]);++ expect(result).toMatchObject({ note: "{not json", environment: "prod" });+ expect(droppedCalls()).toHaveLength(0);+ const undecodedCalls = recordIncrementMock.mock.calls.filter(+ ([stat]) => stat === "langfuse.ingestion.metadata_undecoded",+ );+ expect(undecodedCalls).toHaveLength(1);+ expect(undecodedCalls[0][2]).toEqual({ kind: "truncated_json" });+ });++ it.each([+ {+ case: "current Python major",+ headerSdkName: "python",+ scopeVersion: "4.15.6",+ },+ {+ case: "current JS major",+ headerSdkName: "javascript",+ scopeVersion: "5.11.1",+ },+ {+ case: "non-Langfuse scope",+ headerSdkName: "python",+ scopeVersion: "5.0.0",+ scopeName: "openinference",+ },+ ])("keeps per-key values as sent for the $case", async (params) => {+ const result = await processMetadata(+ "v4",+ perKeyAttributes("observation"),+ params,+ );++ expect(result).toMatchObject({+ environment: '"prod"',+ retries: "3",+ database: '{"host":"localhost","port":5432,"auth":{"user":"app"}}',+ });+ });++ it("falls back to telemetry.sdk.language without a Langfuse SDK header", async () => {+ const result = await processMetadata(+ "v4",+ perKeyAttributes("observation"),+ {+ headerSdkName: "unknown",+ scopeVersion: "6.0.0-beta.1",+ telemetrySdkLanguage: "nodejs",+ },+ );++ expect(result).toMatchObject({+ environment: "prod",+ database: metadata.database,+ });+ });+});+describe("OTel metadata_dropped metric", () => {beforeEach(() => {recordIncrementMock.mockClear();diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex 883cde0b5c..6b6ac015a3 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -27,6 +27,9 @@ import {DEFAULT_TRACE_ENVIRONMENT,LangfuseInternalTraceEnvironment,sanitizeSdkMetricTagValue,+ normalizeIngestionSdkName,+ isStructuredMetadataSdkVersion,+ type IngestionSdkCanonicalName,} from "../";import {@@ -107,6 +110,16 @@ interface MetadataDropContext {dropScope: object;}+// OTel `telemetry.sdk.language` values emitted by the Langfuse SDK runtimes.+const OTEL_SDK_LANGUAGE_TO_SDK_NAME = new Map<+ string,+ IngestionSdkCanonicalName+>([+ ["python", "python"],+ ["nodejs", "javascript"],+ ["webjs", "javascript"],+]);+const DANGEROUS_OTEL_PATH_SEGMENTS = new Set(["__proto__","constructor",@@ -179,6 +192,7 @@ interface CreateTraceEventParams {scopeSpan: any;scopeAttributes: Record<string, unknown>;isLangfuseSDKSpans: boolean;+ decodeMetadataValues: boolean;isRootSpan: boolean;hasTraceUpdates: boolean;parentObservationId: string | null;@@ -415,16 +429,23 @@ export class OtelIngestionProcessor {const isLangfuseSDKSpans =scopeSpan.scope?.name?.startsWith("langfuse-sdk") ?? false;+ const decodeMetadataValues = this.shouldDecodeMetadataValues(+ scopeSpan,+ resourceAttributes,+ );+// Extract metadata from different sourcesconst spanMetadata = this.extractMetadata(spanAttributes,"observation",span,+ decodeMetadataValues,);const traceMetadata = this.extractMetadata(spanAttributes,"trace",span,+ decodeMetadataValues,);const { input, output, filteredAttributes } =@@ -976,10 +997,15 @@ export class OtelIngestionProcessor {? this.parseId(span.parentSpanId?.data ?? span.parentSpanId): null;+ const decodeMetadataValues = this.shouldDecodeMetadataValues(+ scopeSpan,+ resourceAttributes,+ );const spanAttributeMetadata = this.extractMetadata(attributes,"observation",span,+ decodeMetadataValues,);const resourceAttributeMetadata = this.extractMetadata(resourceAttributes,@@ -1010,6 +1036,7 @@ export class OtelIngestionProcessor {scopeSpan,scopeAttributes,isLangfuseSDKSpans,+ decodeMetadataValues,isRootSpan,hasTraceUpdates,parentObservationId,@@ -1052,6 +1079,7 @@ export class OtelIngestionProcessor {scopeSpan,scopeAttributes,isLangfuseSDKSpans,+ decodeMetadataValues,isRootSpan,hasTraceUpdates,span,@@ -1083,7 +1111,12 @@ export class OtelIngestionProcessor {this.extractName(span.name, attributes),metadata: {...resourceAttributeMetadata,- ...this.extractMetadata(attributes, "trace", span),+ ...this.extractMetadata(+ attributes,+ "trace",+ span,+ decodeMetadataValues,+ ),...spanAttributeMetadata,...(isLangfuseSDKSpans ? {} : { attributes: filteredAttributes }),resourceAttributes,@@ -1116,7 +1149,12 @@ export class OtelIngestionProcessor {name: attributes[LangfuseOtelSpanAttributes.TRACE_NAME] as string,metadata: {...resourceAttributeMetadata,- ...this.extractMetadata(attributes, "trace", span),+ ...this.extractMetadata(+ attributes,+ "trace",+ span,+ decodeMetadataValues,+ ),// removed to not remove trace metadata->attributes through subsequent observations// ...(isLangfuseSDKSpans// ? {}@@ -2566,6 +2604,7 @@ export class OtelIngestionProcessor {attributes: Record<string, unknown>,domain: "trace" | "observation",dropScope: object,+ decodeValues = false,): Record<string, unknown> {const metadataKeyPrefix =domain === "observation"@@ -2606,6 +2645,7 @@ export class OtelIngestionProcessor {"ai.telemetry.metadata.tags","ai.telemetry.metadata.langfusePrompt",]),+ decodeValues,});return {@@ -3466,8 +3506,15 @@ export class OtelIngestionProcessor {attributes: Record<string, unknown>;prefixes: string[];excludedKeys?: Set<string>;+ // Set for structured-metadata SDKs, which JSON-encode every value.+ decodeValues?: boolean;}): Record<string, unknown> {- const { attributes, prefixes, excludedKeys = new Set<string>() } = params;+ const {+ attributes,+ prefixes,+ excludedKeys = new Set<string>(),+ decodeValues,+ } = params;const metadata: Record<string, unknown> = {};for (const [key, value] of Object.entries(attributes)) {for (const prefix of prefixes) {@@ -3480,13 +3527,50 @@ export class OtelIngestionProcessor {continue;}- metadata[metadataKey] = value;+ if (!decodeValues || typeof value !== "string") {+ metadata[metadataKey] = value;+ continue;+ }++ // Decoded values match what a metadata blob holds for the key. An+ // undecodable value (e.g. cut by an attribute length limit) is kept as+ // sent so its content is not lost.+ try {+ metadata[metadataKey] = JSON.parse(value);+ } catch {+ metadata[metadataKey] = value;+ recordIncrement("langfuse.ingestion.metadata_undecoded", 1, {+ kind: this.classifyParseFailure(value),+ });+ }}}return metadata;}+ // Langfuse SDK spans from a major that opts into decoded per-key metadata.+ // The language comes from the request header, falling back to the OTel+ // resource for exports relayed without Langfuse headers.+ private shouldDecodeMetadataValues(+ scopeSpan: any,+ resourceAttributes: Record<string, unknown>,+ ): boolean {+ if (!scopeSpan?.scope?.name?.startsWith("langfuse-sdk")) {+ return false;+ }+ const sdkName =+ normalizeIngestionSdkName(this.sdkName) ??+ OTEL_SDK_LANGUAGE_TO_SDK_NAME.get(+ String(resourceAttributes["telemetry.sdk.language"]),+ ) ??+ null;+ return isStructuredMetadataSdkVersion({+ sdkName,+ sdkVersion: scopeSpan.scope.version,+ });+ }+private extractMetadataFromPrefix(params: {attributes: Record<string, unknown>;prefix: string;