🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(ingestion): add sdk/attributeKey/parse-failure-kind tags to metadata_dropped (#16850)
Niklas Semmlerfe6645ea62f7
diff --git a/packages/shared/src/server/ingestion/ingestionAttribution.ts b/packages/shared/src/server/ingestion/ingestionAttribution.tsindex 78ed508fa8..ef03872038 100644--- a/packages/shared/src/server/ingestion/ingestionAttribution.ts+++ b/packages/shared/src/server/ingestion/ingestionAttribution.ts@@ -203,6 +203,42 @@ export const classifyIngestionSdkAttribution = (params: {return "attributed";};+const METRIC_TAG_MAX_LENGTH = 32;++/**+ * Bounds a caller-supplied SDK value for safe use as a StatsD/Datadog metric+ * tag. Ingestion SDK name/version come straight from request headers, so+ * without this a caller could inject oversized or separator-bearing+ * (`,` `|` `:` `=`) tag values and inflate metric cardinality. Strips control+ * characters and tag separators, caps length, and falls back to "unknown".+ */+export const sanitizeSdkMetricTagValue = (+ value: string | null | undefined,+): string => {+ if (!value) {+ return UNKNOWN_INGESTION_SDK_VALUE;+ }+ const sanitized = Array.from(value)+ .filter((character) => {+ const codePoint = character.codePointAt(0) ?? 0;+ if (codePoint <= 31 || codePoint === 127) {+ return false;+ }+ return (+ character !== "," &&+ character !== "|" &&+ character !== ":" &&+ character !== "="+ );+ })+ .join("")+ .trim();+ const bounded = Array.from(sanitized)+ .slice(0, METRIC_TAG_MAX_LENGTH)+ .join("");+ return bounded || UNKNOWN_INGESTION_SDK_VALUE;+};+const sanitizeCallerAttributionValue = (value: string,maxLength: number,diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.tsindex 879cfc9cb4..e8b91d8770 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts@@ -1,12 +1,14 @@/**- * LFE-14342: `langfuse.ingestion.metadata_dropped` counter at the OTel- * metadata drop site (`parseMetadataAttribute` drop branches).+ * `langfuse.ingestion.metadata_dropped` counter at the OTel metadata drop+ * site (`parseMetadataAttribute` drop branches).** Spec under test:* - Counter name: langfuse.ingestion.metadata_dropped* - Tags: reason ∈ {non_object_top_level, parse_failure, primitive},* source = otel, domain ∈ {trace, observation}, projectId (low- * cardinality: only projects emitting malformed metadata appear)+ * cardinality: only projects emitting malformed metadata appear),+ * attributeKey (closed set of Langfuse constants), sdkName, sdkVersion,+ * and — for parse_failure only — kind (value-shape sub-classification)* - No behavior change to what the processor returns* - Dotted-key metadata (langfuse.*.metadata.foo) stays increment-free*@@ -112,9 +114,12 @@ const expectDropTags = (expect(tags).toEqual(expect.objectContaining(expected));// projectId is a metric tag (low cardinality) for tenant attribution.expect(tags?.projectId).toBe(PROJECT_ID);+ // sdkName/sdkVersion attribute the emitting client.+ expect(tags?.sdkName).toBe("python");+ expect(tags?.sdkVersion).toBe("3.8.1");};-describe("OTel metadata_dropped metric (LFE-14342)", () => {+describe("OTel metadata_dropped metric", () => {beforeEach(() => {recordIncrementMock.mockClear();});@@ -745,3 +750,100 @@ describe("OTel reconstructed array drop telemetry", () => {}});});++describe("metadata_dropped attribution tags (parse_failure kind, attributeKey)", () => {+ beforeEach(() => {+ recordIncrementMock.mockClear();+ });++ const singleDropTags = () => {+ const calls = recordIncrementMock.mock.calls.filter(+ ([stat]) => stat === METRIC,+ );+ expect(calls).toHaveLength(1);+ return (calls[0] as [string, number, Record<string, string>])[2];+ };++ // Each value fails JSON.parse and must be sub-classified by its shape.+ // The values are synthetic and carry no real user content.+ const kindCases: Array<{ name: string; value: string; kind: string }> = [+ {+ name: "python dict repr",+ value: "{'user': 'x', 'ok': True}",+ kind: "python_repr",+ },+ { name: "bare True token", value: "True", kind: "python_repr" },+ { name: "unquoted string", value: "in progress", kind: "unquoted_string" },+ {+ name: "truncated object",+ value: '{"a":"long va',+ kind: "truncated_json",+ },+ {+ name: "loose json trailing comma",+ value: '{"a":1,}',+ kind: "loose_json",+ },+ { name: "empty string on compat key", value: "", kind: "empty" },+ ];++ for (const { name, value, kind } of kindCases) {+ it(`classifies ${name} as kind=${kind}`, async () => {+ // Empty is only reachable on the compat key (a falsy primary value+ // survives the `||` fallback); non-empty values sit on the primary key.+ const attrKey =+ value === "" ? "langfuse.metadata" : "langfuse.observation.metadata";+ await createProcessor().processToIngestionEvents(+ buildBatch([{ key: attrKey, value: { stringValue: value } }]),+ );++ const tags = singleDropTags();+ expect(tags.reason).toBe("parse_failure");+ expect(tags.kind).toBe(kind);+ expect(tags.attributeKey).toBe(attrKey);+ expect(tags.sdkName).toBe("python");+ expect(tags.sdkVersion).toBe("3.8.1");+ });+ }++ it("omits kind for a non-parse_failure drop (primitive)", async () => {+ await createProcessor().processToIngestionEvents(+ buildBatch([+ { key: "langfuse.observation.metadata", value: { intValue: 42 } },+ ]),+ );+ const tags = singleDropTags();+ expect(tags.reason).toBe("primitive");+ expect(tags.kind).toBeUndefined();+ });++ it("sanitizes and bounds attacker-controlled sdkName/sdkVersion tags", async () => {+ // sdkName/sdkVersion come from raw request headers; a caller must not be+ // able to inject tag separators, control chars, or oversized values.+ const processor = new OtelIngestionProcessor({+ projectId: PROJECT_ID,+ publicKey: "pk-test",+ sdkName: "evil,name|with:sep=chars",+ sdkVersion: `1.0\n${"x".repeat(100)}`,+ });+ await processor.processToIngestionEvents(+ buildBatch([+ {+ key: "langfuse.observation.metadata",+ value: { stringValue: "{bad" },+ },+ ]),+ );++ const tags = singleDropTags();+ for (const key of ["sdkName", "sdkVersion"] as const) {+ expect(tags[key]).not.toMatch(/[,|:=]/);+ expect(tags[key].length).toBeLessThanOrEqual(32);+ for (const character of tags[key]) {+ const codePoint = character.codePointAt(0) ?? 0;+ expect(codePoint).toBeGreaterThan(31);+ expect(codePoint).not.toBe(127);+ }+ }+ });+});diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex 7b96198f2a..98da76e795 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -25,6 +25,7 @@ import {normalizeEnvironment,DEFAULT_TRACE_ENVIRONMENT,LangfuseInternalTraceEnvironment,+ sanitizeSdkMetricTagValue,} from "../";import {@@ -2510,9 +2511,11 @@ export class OtelIngestionProcessor {// survive the fallback and are counted by the parser itself.const primaryValue = attributes[metadataKeyPrefix];if (primaryValue !== undefined && primaryValue !== null && !primaryValue) {+ const isString = typeof primaryValue === "string";this.recordMetadataDropped(- typeof primaryValue === "string" ? "parse_failure" : "primitive",+ isString ? "parse_failure" : "primitive",{ domain, attributeKey: metadataKeyPrefix, dropScope },+ isString ? this.classifyParseFailure(primaryValue) : undefined,);}@@ -3275,6 +3278,7 @@ export class OtelIngestionProcessor {private recordMetadataDropped(reason: string,context: MetadataDropContext,+ kind?: string,): void {const { domain, attributeKey, dropScope } = context;let seen = this.reportedMetadataDrops.get(dropScope);@@ -3288,12 +3292,46 @@ export class OtelIngestionProcessor {}seen.add(dedupeKey);- recordIncrement("langfuse.ingestion.metadata_dropped", 1, {+ // attributeKey is a closed set of Langfuse-defined attribute names+ // (never a user-supplied key); sdkName/sdkVersion attribute the emitting+ // client and are sanitized because they originate from raw request+ // headers; kind sub-classifies parse_failure by the value's shape. None+ // of these carry the dropped value itself.+ const tags: Record<string, string> = {reason,source: "otel",domain,projectId: this.projectId,- });+ attributeKey,+ sdkName: sanitizeSdkMetricTagValue(this.sdkName),+ sdkVersion: sanitizeSdkMetricTagValue(this.sdkVersion),+ };+ if (kind) {+ tags.kind = kind;+ }+ recordIncrement("langfuse.ingestion.metadata_dropped", 1, tags);+ }++ // Sub-classifies a JSON.parse failure by the failing string's shape,+ // reading only bounded head/tail windows — never re-parses or copies the+ // (possibly multi-MB) value, and never logs its content. Slicing first+ // keeps the trim/regex work off the full string.+ private classifyParseFailure(value: string): string {+ const head = value.slice(0, 64).trimStart();+ if (head.length === 0) {+ return "empty";+ }+ // Python `str(dict)` / `str(list)` — single-quoted, or bare True/False/None.+ if (/^[{[]\s*'/.test(head) || /^(True|False|None)\b/.test(head)) {+ return "python_repr";+ }+ const first = head[0];+ if (first === "{" || first === "[") {+ const expectedClose = first === "{" ? "}" : "]";+ const last = value.slice(-64).trimEnd().slice(-1);+ return last === expectedClose ? "loose_json" : "truncated_json";+ }+ return "unquoted_string";}private parseMetadataAttribute(@@ -3313,7 +3351,11 @@ export class OtelIngestionProcessor {this.recordMetadataDropped("non_object_top_level", context);return {};} catch {- this.recordMetadataDropped("parse_failure", context);+ this.recordMetadataDropped(+ "parse_failure",+ context,+ this.classifyParseFailure(value),+ );return {};}}diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex 3053af69b4..7f077af253 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -54,6 +54,7 @@ import {normalizeToolsForObservation,hasNoEvalConfigsCache,buildClickHouseLogComment,+ sanitizeSdkMetricTagValue,type IngestionAttribution,type PricingTierMatchAttributes,} from "@langfuse/shared/src/server";@@ -729,6 +730,8 @@ export class IngestionService {source: "api",domain: "score",projectId,+ sdkName: sanitizeSdkMetricTagValue(attribution.ingestionSdkName),+ sdkVersion: sanitizeSdkMetricTagValue(attribution.ingestionSdkVersion),});}diff --git a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.tsindex 6d3334834d..6bdce7a235 100644--- a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts+++ b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts@@ -1,6 +1,6 @@/**- * LFE-14345: `langfuse.ingestion.metadata_dropped` counter at the score- * validation drop site (catch in processScoreEventList).+ * `langfuse.ingestion.metadata_dropped` counter at the score validation+ * drop site (catch in processScoreEventList).** Spec under test:* - The catch that silently filters score events on validation failure@@ -10,7 +10,8 @@* events in the same batch are still written, unexpected errors still* reject the batch (and must NOT emit the drop counter — a rejected* batch is retried, not silently lost).- * - projectId is a metric tag (low cardinality) for tenant attribution.+ * - projectId, sdkName, sdkVersion are metric tags (low cardinality) for+ * tenant and client attribution.*/import { expect, describe, it, vi, beforeEach } from "vitest";@@ -62,6 +63,8 @@ const expectScoreDropTags = (call: unknown[]) => {source: "api",domain: "score",projectId: PROJECT_ID,+ sdkName: "langfuse-test",+ sdkVersion: "0.0.0",}),);};@@ -116,7 +119,7 @@ const processScores = (},});-describe("score validation drop metric (LFE-14345)", () => {+describe("score validation drop metric", () => {beforeEach(() => {mocks.recordIncrement.mockClear();mocks.validateAndInflateScoreOverride = undefined;