Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ingestion): decode per-key OTel metadata like a blob for next SDK majors (#18436)

Niklas Semmler5979790e2c4a
diff --git a/packages/shared/src/server/ingestion/ingestionAttribution.ts b/packages/shared/src/server/ingestion/ingestionAttribution.ts
index ef03872038..269a1d452c 100644
--- a/packages/shared/src/server/ingestion/ingestionAttribution.ts
+++ b/packages/shared/src/server/ingestion/ingestionAttribution.ts
@@ -25,6 +25,13 @@ export const LANGFUSE_SDK_LATEST_MAJOR = {
export type IngestionSdkCanonicalName = keyof typeof LANGFUSE_SDK_LATEST_MAJOR;
+// First SDK majors that JSON-encode every per-key metadata value. Ingestion
+// decodes those values so they are stored exactly like a metadata blob.
+const LANGFUSE_SDK_STRUCTURED_METADATA_MIN_MAJOR = {
+ python: 5,
+ javascript: 6,
+} as const satisfies Record<IngestionSdkCanonicalName, number>;
+
export type IngestionSdkUpgradeStatus =
| "current"
| "outdated_major"
@@ -134,6 +141,22 @@ export const extractBaseIngestionSdkVersion = (sdkVersion: string): string => {
return version;
};
+export const isStructuredMetadataSdkVersion = (params: {
+ sdkName: IngestionSdkCanonicalName | null;
+ sdkVersion: string | null | undefined;
+}): boolean => {
+ if (!params.sdkName || !params.sdkVersion) return false;
+
+ const parsedVersion = parseVersionString(
+ extractBaseIngestionSdkVersion(params.sdkVersion),
+ );
+ return (
+ parsedVersion !== null &&
+ parsedVersion.major >=
+ LANGFUSE_SDK_STRUCTURED_METADATA_MIN_MAJOR[params.sdkName]
+ );
+};
+
export const classifyIngestionSdkVersion = (params: {
sdkName: string | null | undefined;
sdkVersion: string | null | undefined;
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
index 2c82dd4d64..79593183d9 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
@@ -44,6 +44,7 @@ import {
type ResourceSpan,
} from "./OtelIngestionProcessor";
import * as serverBarrel from "../index";
+import { flattenJsonToPathArrays } from "./utils";
const METRIC = "langfuse.ingestion.metadata_dropped";
const ARRAY_ATTRIBUTE_DROPPED_METRIC =
@@ -417,6 +418,201 @@ describe("raw OTLP empty attribute values", () => {
});
});
+describe("per-key metadata values from structured-metadata SDK majors", () => {
+ const metadata = {
+ environment: "prod",
+ version: "1.0",
+ digits: "42",
+ label: "",
+ retries: 3,
+ ratio: 1.5,
+ enabled: true,
+ missing: null,
+ database: { host: "localhost", port: 5432, auth: { user: "app" } },
+ steps: [{ name: "web_fetch", args: { url: "https://example.com" } }],
+ empty: {},
+ };
+
+ const blobAttributes = (
+ domain: "observation" | "trace",
+ space?: number,
+ ): OtelAttribute[] => [
+ {
+ key: `langfuse.${domain}.metadata`,
+ value: { stringValue: JSON.stringify(metadata, null, space) },
+ },
+ ];
+
+ // Structured-metadata SDKs JSON-encode every value, strings included.
+ const perKeyAttributes = (
+ domain: "observation" | "trace",
+ space?: number,
+ ): OtelAttribute[] =>
+ Object.entries(metadata).map(([key, value]) => ({
+ key: `langfuse.${domain}.metadata.${key}`,
+ value: { stringValue: JSON.stringify(value, null, space) },
+ }));
+
+ const processMetadata = async (
+ path: "v3" | "v4",
+ attributes: OtelAttribute[],
+ params: {
+ domain?: "observation" | "trace";
+ // "child" sends the attributes on a child span of an attribute-less root.
+ span?: "root" | "child";
+ headerSdkName?: string;
+ scopeName?: string;
+ scopeVersion?: string;
+ telemetrySdkLanguage?: string;
+ } = {},
+ ) => {
+ const scopeVersion = params.scopeVersion ?? "5.0.0";
+ const batch = buildBatch(params.span === "child" ? [] : attributes);
+ if (params.span === "child") {
+ const spans = batch[0].scopeSpans![0].spans!;
+ spans.push({
+ ...spans[0],
+ spanId: Buffer.from("fedcba9876543210", "hex"),
+ parentSpanId: spans[0].spanId,
+ name: "child-span",
+ attributes: [...spans[0].attributes!, ...attributes],
+ });
+ }
+ const scope = batch[0].scopeSpans![0].scope!;
+ scope.name = params.scopeName ?? "langfuse-sdk";
+ scope.version = scopeVersion;
+ if (params.telemetrySdkLanguage) {
+ batch[0].resource!.attributes!.push({
+ key: "telemetry.sdk.language",
+ value: { stringValue: params.telemetrySdkLanguage },
+ });
+ }
+ const processor = new OtelIngestionProcessor({
+ projectId: PROJECT_ID,
+ publicKey: "pk-test",
+ sdkName: params.headerSdkName ?? "python",
+ sdkVersion: scopeVersion,
+ });
+ if (path === "v4") {
+ return processor.processToEvent(batch).at(-1)?.metadata as Record<
+ string,
+ unknown
+ >;
+ }
+ // v3 writes trace metadata to the trace, observation metadata to the span.
+ // The last matching event belongs to the span carrying the attributes.
+ const eventType =
+ params.domain === "trace" ? "trace-create" : "span-create";
+ const event = (await processor.processToIngestionEvents(batch))
+ .filter((event) => event.type === eventType)
+ .at(-1);
+ return (event?.body as { metadata?: Record<string, unknown> } | undefined)
+ ?.metadata as Record<string, unknown>;
+ };
+
+ it.each([
+ { path: "v3", domain: "observation", span: "root", encoding: "compact" },
+ { path: "v3", domain: "trace", span: "root", encoding: "compact" },
+ { path: "v3", domain: "trace", span: "child", encoding: "compact" },
+ { path: "v4", domain: "observation", span: "root", encoding: "compact" },
+ { path: "v4", domain: "trace", span: "root", encoding: "compact" },
+ { path: "v4", domain: "trace", span: "child", encoding: "compact" },
+ { path: "v3", domain: "observation", span: "root", encoding: "indented" },
+ { path: "v4", domain: "observation", span: "root", encoding: "indented" },
+ ] as const)(
+ "stores per-key values exactly like a blob ($path, $domain on $span span, $encoding JSON)",
+ async ({ path, domain, span, encoding }) => {
+ const space = encoding === "indented" ? 2 : undefined;
+ const fromBlob = await processMetadata(
+ path,
+ blobAttributes(domain, space),
+ { domain, span },
+ );
+ const fromPerKey = await processMetadata(
+ path,
+ perKeyAttributes(domain, space),
+ { domain, span },
+ );
+
+ expect(fromBlob).toMatchObject(metadata);
+ expect(fromPerKey).toEqual(fromBlob);
+ expect(flattenJsonToPathArrays(fromPerKey)).toEqual(
+ flattenJsonToPathArrays(fromBlob),
+ );
+ },
+ );
+
+ it("keeps a per-key value that is not valid JSON as sent and counts it", async () => {
+ recordIncrementMock.mockClear();
+ const result = await processMetadata("v4", [
+ {
+ key: "langfuse.observation.metadata.note",
+ value: { stringValue: "{not json" },
+ },
+ {
+ key: "langfuse.observation.metadata.environment",
+ value: { stringValue: '"prod"' },
+ },
+ ]);
+
+ expect(result).toMatchObject({ note: "{not json", environment: "prod" });
+ expect(droppedCalls()).toHaveLength(0);
+ const undecodedCalls = recordIncrementMock.mock.calls.filter(
+ ([stat]) => stat === "langfuse.ingestion.metadata_undecoded",
+ );
+ expect(undecodedCalls).toHaveLength(1);
+ expect(undecodedCalls[0][2]).toEqual({ kind: "truncated_json" });
+ });
+
+ it.each([
+ {
+ case: "current Python major",
+ headerSdkName: "python",
+ scopeVersion: "4.15.6",
+ },
+ {
+ case: "current JS major",
+ headerSdkName: "javascript",
+ scopeVersion: "5.11.1",
+ },
+ {
+ case: "non-Langfuse scope",
+ headerSdkName: "python",
+ scopeVersion: "5.0.0",
+ scopeName: "openinference",
+ },
+ ])("keeps per-key values as sent for the $case", async (params) => {
+ const result = await processMetadata(
+ "v4",
+ perKeyAttributes("observation"),
+ params,
+ );
+
+ expect(result).toMatchObject({
+ environment: '"prod"',
+ retries: "3",
+ database: '{"host":"localhost","port":5432,"auth":{"user":"app"}}',
+ });
+ });
+
+ it("falls back to telemetry.sdk.language without a Langfuse SDK header", async () => {
+ const result = await processMetadata(
+ "v4",
+ perKeyAttributes("observation"),
+ {
+ headerSdkName: "unknown",
+ scopeVersion: "6.0.0-beta.1",
+ telemetrySdkLanguage: "nodejs",
+ },
+ );
+
+ expect(result).toMatchObject({
+ environment: "prod",
+ database: metadata.database,
+ });
+ });
+});
+
describe("OTel metadata_dropped metric", () => {
beforeEach(() => {
recordIncrementMock.mockClear();
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index 883cde0b5c..6b6ac015a3 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -27,6 +27,9 @@ import {
DEFAULT_TRACE_ENVIRONMENT,
LangfuseInternalTraceEnvironment,
sanitizeSdkMetricTagValue,
+ normalizeIngestionSdkName,
+ isStructuredMetadataSdkVersion,
+ type IngestionSdkCanonicalName,
} from "../";
import {
@@ -107,6 +110,16 @@ interface MetadataDropContext {
dropScope: object;
}
+// OTel `telemetry.sdk.language` values emitted by the Langfuse SDK runtimes.
+const OTEL_SDK_LANGUAGE_TO_SDK_NAME = new Map<
+ string,
+ IngestionSdkCanonicalName
+>([
+ ["python", "python"],
+ ["nodejs", "javascript"],
+ ["webjs", "javascript"],
+]);
+
const DANGEROUS_OTEL_PATH_SEGMENTS = new Set([
"__proto__",
"constructor",
@@ -179,6 +192,7 @@ interface CreateTraceEventParams {
scopeSpan: any;
scopeAttributes: Record<string, unknown>;
isLangfuseSDKSpans: boolean;
+ decodeMetadataValues: boolean;
isRootSpan: boolean;
hasTraceUpdates: boolean;
parentObservationId: string | null;
@@ -415,16 +429,23 @@ export class OtelIngestionProcessor {
const isLangfuseSDKSpans =
scopeSpan.scope?.name?.startsWith("langfuse-sdk") ?? false;
+ const decodeMetadataValues = this.shouldDecodeMetadataValues(
+ scopeSpan,
+ resourceAttributes,
+ );
+
// Extract metadata from different sources
const spanMetadata = this.extractMetadata(
spanAttributes,
"observation",
span,
+ decodeMetadataValues,
);
const traceMetadata = this.extractMetadata(
spanAttributes,
"trace",
span,
+ decodeMetadataValues,
);
const { input, output, filteredAttributes } =
@@ -976,10 +997,15 @@ export class OtelIngestionProcessor {
? this.parseId(span.parentSpanId?.data ?? span.parentSpanId)
: null;
+ const decodeMetadataValues = this.shouldDecodeMetadataValues(
+ scopeSpan,
+ resourceAttributes,
+ );
const spanAttributeMetadata = this.extractMetadata(
attributes,
"observation",
span,
+ decodeMetadataValues,
);
const resourceAttributeMetadata = this.extractMetadata(
resourceAttributes,
@@ -1010,6 +1036,7 @@ export class OtelIngestionProcessor {
scopeSpan,
scopeAttributes,
isLangfuseSDKSpans,
+ decodeMetadataValues,
isRootSpan,
hasTraceUpdates,
parentObservationId,
@@ -1052,6 +1079,7 @@ export class OtelIngestionProcessor {
scopeSpan,
scopeAttributes,
isLangfuseSDKSpans,
+ decodeMetadataValues,
isRootSpan,
hasTraceUpdates,
span,
@@ -1083,7 +1111,12 @@ export class OtelIngestionProcessor {
this.extractName(span.name, attributes),
metadata: {
...resourceAttributeMetadata,
- ...this.extractMetadata(attributes, "trace", span),
+ ...this.extractMetadata(
+ attributes,
+ "trace",
+ span,
+ decodeMetadataValues,
+ ),
...spanAttributeMetadata,
...(isLangfuseSDKSpans ? {} : { attributes: filteredAttributes }),
resourceAttributes,
@@ -1116,7 +1149,12 @@ export class OtelIngestionProcessor {
name: attributes[LangfuseOtelSpanAttributes.TRACE_NAME] as string,
metadata: {
...resourceAttributeMetadata,
- ...this.extractMetadata(attributes, "trace", span),
+ ...this.extractMetadata(
+ attributes,
+ "trace",
+ span,
+ decodeMetadataValues,
+ ),
// removed to not remove trace metadata->attributes through subsequent observations
// ...(isLangfuseSDKSpans
// ? {}
@@ -2566,6 +2604,7 @@ export class OtelIngestionProcessor {
attributes: Record<string, unknown>,
domain: "trace" | "observation",
dropScope: object,
+ decodeValues = false,
): Record<string, unknown> {
const metadataKeyPrefix =
domain === "observation"
@@ -2606,6 +2645,7 @@ export class OtelIngestionProcessor {
"ai.telemetry.metadata.tags",
"ai.telemetry.metadata.langfusePrompt",
]),
+ decodeValues,
});
return {
@@ -3466,8 +3506,15 @@ export class OtelIngestionProcessor {
attributes: Record<string, unknown>;
prefixes: string[];
excludedKeys?: Set<string>;
+ // Set for structured-metadata SDKs, which JSON-encode every value.
+ decodeValues?: boolean;
}): Record<string, unknown> {
- const { attributes, prefixes, excludedKeys = new Set<string>() } = params;
+ const {
+ attributes,
+ prefixes,
+ excludedKeys = new Set<string>(),
+ decodeValues,
+ } = params;
const metadata: Record<string, unknown> = {};
for (const [key, value] of Object.entries(attributes)) {
for (const prefix of prefixes) {
@@ -3480,13 +3527,50 @@ export class OtelIngestionProcessor {
continue;
}
- metadata[metadataKey] = value;
+ if (!decodeValues || typeof value !== "string") {
+ metadata[metadataKey] = value;
+ continue;
+ }
+
+ // Decoded values match what a metadata blob holds for the key. An
+ // undecodable value (e.g. cut by an attribute length limit) is kept as
+ // sent so its content is not lost.
+ try {
+ metadata[metadataKey] = JSON.parse(value);
+ } catch {
+ metadata[metadataKey] = value;
+ recordIncrement("langfuse.ingestion.metadata_undecoded", 1, {
+ kind: this.classifyParseFailure(value),
+ });
+ }
}
}
return metadata;
}
+ // Langfuse SDK spans from a major that opts into decoded per-key metadata.
+ // The language comes from the request header, falling back to the OTel
+ // resource for exports relayed without Langfuse headers.
+ private shouldDecodeMetadataValues(
+ scopeSpan: any,
+ resourceAttributes: Record<string, unknown>,
+ ): boolean {
+ if (!scopeSpan?.scope?.name?.startsWith("langfuse-sdk")) {
+ return false;
+ }
+ const sdkName =
+ normalizeIngestionSdkName(this.sdkName) ??
+ OTEL_SDK_LANGUAGE_TO_SDK_NAME.get(
+ String(resourceAttributes["telemetry.sdk.language"]),
+ ) ??
+ null;
+ return isStructuredMetadataSdkVersion({
+ sdkName,
+ sdkVersion: scopeSpan.scope.version,
+ });
+ }
+
private extractMetadataFromPrefix(params: {
attributes: Record<string, unknown>;
prefix: string;