Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(ingestion): add sdk/attributeKey/parse-failure-kind tags to metadata_dropped (#16850)

Niklas Semmlerfe6645ea62f7
diff --git a/packages/shared/src/server/ingestion/ingestionAttribution.ts b/packages/shared/src/server/ingestion/ingestionAttribution.ts
index 78ed508fa8..ef03872038 100644
--- a/packages/shared/src/server/ingestion/ingestionAttribution.ts
+++ b/packages/shared/src/server/ingestion/ingestionAttribution.ts
@@ -203,6 +203,42 @@ export const classifyIngestionSdkAttribution = (params: {
return "attributed";
};
+const METRIC_TAG_MAX_LENGTH = 32;
+
+/**
+ * Bounds a caller-supplied SDK value for safe use as a StatsD/Datadog metric
+ * tag. Ingestion SDK name/version come straight from request headers, so
+ * without this a caller could inject oversized or separator-bearing
+ * (`,` `|` `:` `=`) tag values and inflate metric cardinality. Strips control
+ * characters and tag separators, caps length, and falls back to "unknown".
+ */
+export const sanitizeSdkMetricTagValue = (
+ value: string | null | undefined,
+): string => {
+ if (!value) {
+ return UNKNOWN_INGESTION_SDK_VALUE;
+ }
+ const sanitized = Array.from(value)
+ .filter((character) => {
+ const codePoint = character.codePointAt(0) ?? 0;
+ if (codePoint <= 31 || codePoint === 127) {
+ return false;
+ }
+ return (
+ character !== "," &&
+ character !== "|" &&
+ character !== ":" &&
+ character !== "="
+ );
+ })
+ .join("")
+ .trim();
+ const bounded = Array.from(sanitized)
+ .slice(0, METRIC_TAG_MAX_LENGTH)
+ .join("");
+ return bounded || UNKNOWN_INGESTION_SDK_VALUE;
+};
+
const sanitizeCallerAttributionValue = (
value: string,
maxLength: number,
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
index 879cfc9cb4..e8b91d8770 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
@@ -1,12 +1,14 @@
/**
- * LFE-14342: `langfuse.ingestion.metadata_dropped` counter at the OTel
- * metadata drop site (`parseMetadataAttribute` drop branches).
+ * `langfuse.ingestion.metadata_dropped` counter at the OTel metadata drop
+ * site (`parseMetadataAttribute` drop branches).
*
* Spec under test:
* - Counter name: langfuse.ingestion.metadata_dropped
* - Tags: reason ∈ {non_object_top_level, parse_failure, primitive},
* source = otel, domain ∈ {trace, observation}, projectId (low
- * cardinality: only projects emitting malformed metadata appear)
+ * cardinality: only projects emitting malformed metadata appear),
+ * attributeKey (closed set of Langfuse constants), sdkName, sdkVersion,
+ * and — for parse_failure only — kind (value-shape sub-classification)
* - No behavior change to what the processor returns
* - Dotted-key metadata (langfuse.*.metadata.foo) stays increment-free
*
@@ -112,9 +114,12 @@ const expectDropTags = (
expect(tags).toEqual(expect.objectContaining(expected));
// projectId is a metric tag (low cardinality) for tenant attribution.
expect(tags?.projectId).toBe(PROJECT_ID);
+ // sdkName/sdkVersion attribute the emitting client.
+ expect(tags?.sdkName).toBe("python");
+ expect(tags?.sdkVersion).toBe("3.8.1");
};
-describe("OTel metadata_dropped metric (LFE-14342)", () => {
+describe("OTel metadata_dropped metric", () => {
beforeEach(() => {
recordIncrementMock.mockClear();
});
@@ -745,3 +750,100 @@ describe("OTel reconstructed array drop telemetry", () => {
}
});
});
+
+describe("metadata_dropped attribution tags (parse_failure kind, attributeKey)", () => {
+ beforeEach(() => {
+ recordIncrementMock.mockClear();
+ });
+
+ const singleDropTags = () => {
+ const calls = recordIncrementMock.mock.calls.filter(
+ ([stat]) => stat === METRIC,
+ );
+ expect(calls).toHaveLength(1);
+ return (calls[0] as [string, number, Record<string, string>])[2];
+ };
+
+ // Each value fails JSON.parse and must be sub-classified by its shape.
+ // The values are synthetic and carry no real user content.
+ const kindCases: Array<{ name: string; value: string; kind: string }> = [
+ {
+ name: "python dict repr",
+ value: "{'user': 'x', 'ok': True}",
+ kind: "python_repr",
+ },
+ { name: "bare True token", value: "True", kind: "python_repr" },
+ { name: "unquoted string", value: "in progress", kind: "unquoted_string" },
+ {
+ name: "truncated object",
+ value: '{"a":"long va',
+ kind: "truncated_json",
+ },
+ {
+ name: "loose json trailing comma",
+ value: '{"a":1,}',
+ kind: "loose_json",
+ },
+ { name: "empty string on compat key", value: "", kind: "empty" },
+ ];
+
+ for (const { name, value, kind } of kindCases) {
+ it(`classifies ${name} as kind=${kind}`, async () => {
+ // Empty is only reachable on the compat key (a falsy primary value
+ // survives the `||` fallback); non-empty values sit on the primary key.
+ const attrKey =
+ value === "" ? "langfuse.metadata" : "langfuse.observation.metadata";
+ await createProcessor().processToIngestionEvents(
+ buildBatch([{ key: attrKey, value: { stringValue: value } }]),
+ );
+
+ const tags = singleDropTags();
+ expect(tags.reason).toBe("parse_failure");
+ expect(tags.kind).toBe(kind);
+ expect(tags.attributeKey).toBe(attrKey);
+ expect(tags.sdkName).toBe("python");
+ expect(tags.sdkVersion).toBe("3.8.1");
+ });
+ }
+
+ it("omits kind for a non-parse_failure drop (primitive)", async () => {
+ await createProcessor().processToIngestionEvents(
+ buildBatch([
+ { key: "langfuse.observation.metadata", value: { intValue: 42 } },
+ ]),
+ );
+ const tags = singleDropTags();
+ expect(tags.reason).toBe("primitive");
+ expect(tags.kind).toBeUndefined();
+ });
+
+ it("sanitizes and bounds attacker-controlled sdkName/sdkVersion tags", async () => {
+ // sdkName/sdkVersion come from raw request headers; a caller must not be
+ // able to inject tag separators, control chars, or oversized values.
+ const processor = new OtelIngestionProcessor({
+ projectId: PROJECT_ID,
+ publicKey: "pk-test",
+ sdkName: "evil,name|with:sep=chars",
+ sdkVersion: `1.0\n${"x".repeat(100)}`,
+ });
+ await processor.processToIngestionEvents(
+ buildBatch([
+ {
+ key: "langfuse.observation.metadata",
+ value: { stringValue: "{bad" },
+ },
+ ]),
+ );
+
+ const tags = singleDropTags();
+ for (const key of ["sdkName", "sdkVersion"] as const) {
+ expect(tags[key]).not.toMatch(/[,|:=]/);
+ expect(tags[key].length).toBeLessThanOrEqual(32);
+ for (const character of tags[key]) {
+ const codePoint = character.codePointAt(0) ?? 0;
+ expect(codePoint).toBeGreaterThan(31);
+ expect(codePoint).not.toBe(127);
+ }
+ }
+ });
+});
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index 7b96198f2a..98da76e795 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -25,6 +25,7 @@ import {
normalizeEnvironment,
DEFAULT_TRACE_ENVIRONMENT,
LangfuseInternalTraceEnvironment,
+ sanitizeSdkMetricTagValue,
} from "../";
import {
@@ -2510,9 +2511,11 @@ export class OtelIngestionProcessor {
// survive the fallback and are counted by the parser itself.
const primaryValue = attributes[metadataKeyPrefix];
if (primaryValue !== undefined && primaryValue !== null && !primaryValue) {
+ const isString = typeof primaryValue === "string";
this.recordMetadataDropped(
- typeof primaryValue === "string" ? "parse_failure" : "primitive",
+ isString ? "parse_failure" : "primitive",
{ domain, attributeKey: metadataKeyPrefix, dropScope },
+ isString ? this.classifyParseFailure(primaryValue) : undefined,
);
}
@@ -3275,6 +3278,7 @@ export class OtelIngestionProcessor {
private recordMetadataDropped(
reason: string,
context: MetadataDropContext,
+ kind?: string,
): void {
const { domain, attributeKey, dropScope } = context;
let seen = this.reportedMetadataDrops.get(dropScope);
@@ -3288,12 +3292,46 @@ export class OtelIngestionProcessor {
}
seen.add(dedupeKey);
- recordIncrement("langfuse.ingestion.metadata_dropped", 1, {
+ // attributeKey is a closed set of Langfuse-defined attribute names
+ // (never a user-supplied key); sdkName/sdkVersion attribute the emitting
+ // client and are sanitized because they originate from raw request
+ // headers; kind sub-classifies parse_failure by the value's shape. None
+ // of these carry the dropped value itself.
+ const tags: Record<string, string> = {
reason,
source: "otel",
domain,
projectId: this.projectId,
- });
+ attributeKey,
+ sdkName: sanitizeSdkMetricTagValue(this.sdkName),
+ sdkVersion: sanitizeSdkMetricTagValue(this.sdkVersion),
+ };
+ if (kind) {
+ tags.kind = kind;
+ }
+ recordIncrement("langfuse.ingestion.metadata_dropped", 1, tags);
+ }
+
+ // Sub-classifies a JSON.parse failure by the failing string's shape,
+ // reading only bounded head/tail windows — never re-parses or copies the
+ // (possibly multi-MB) value, and never logs its content. Slicing first
+ // keeps the trim/regex work off the full string.
+ private classifyParseFailure(value: string): string {
+ const head = value.slice(0, 64).trimStart();
+ if (head.length === 0) {
+ return "empty";
+ }
+ // Python `str(dict)` / `str(list)` — single-quoted, or bare True/False/None.
+ if (/^[{[]\s*'/.test(head) || /^(True|False|None)\b/.test(head)) {
+ return "python_repr";
+ }
+ const first = head[0];
+ if (first === "{" || first === "[") {
+ const expectedClose = first === "{" ? "}" : "]";
+ const last = value.slice(-64).trimEnd().slice(-1);
+ return last === expectedClose ? "loose_json" : "truncated_json";
+ }
+ return "unquoted_string";
}
private parseMetadataAttribute(
@@ -3313,7 +3351,11 @@ export class OtelIngestionProcessor {
this.recordMetadataDropped("non_object_top_level", context);
return {};
} catch {
- this.recordMetadataDropped("parse_failure", context);
+ this.recordMetadataDropped(
+ "parse_failure",
+ context,
+ this.classifyParseFailure(value),
+ );
return {};
}
}
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 3053af69b4..7f077af253 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -54,6 +54,7 @@ import {
normalizeToolsForObservation,
hasNoEvalConfigsCache,
buildClickHouseLogComment,
+ sanitizeSdkMetricTagValue,
type IngestionAttribution,
type PricingTierMatchAttributes,
} from "@langfuse/shared/src/server";
@@ -729,6 +730,8 @@ export class IngestionService {
source: "api",
domain: "score",
projectId,
+ sdkName: sanitizeSdkMetricTagValue(attribution.ingestionSdkName),
+ sdkVersion: sanitizeSdkMetricTagValue(attribution.ingestionSdkVersion),
});
}
diff --git a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts
index 6d3334834d..6bdce7a235 100644
--- a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts
@@ -1,6 +1,6 @@
/**
- * LFE-14345: `langfuse.ingestion.metadata_dropped` counter at the score
- * validation drop site (catch in processScoreEventList).
+ * `langfuse.ingestion.metadata_dropped` counter at the score validation
+ * drop site (catch in processScoreEventList).
*
* Spec under test:
* - The catch that silently filters score events on validation failure
@@ -10,7 +10,8 @@
* events in the same batch are still written, unexpected errors still
* reject the batch (and must NOT emit the drop counter — a rejected
* batch is retried, not silently lost).
- * - projectId is a metric tag (low cardinality) for tenant attribution.
+ * - projectId, sdkName, sdkVersion are metric tags (low cardinality) for
+ * tenant and client attribution.
*/
import { expect, describe, it, vi, beforeEach } from "vitest";
@@ -62,6 +63,8 @@ const expectScoreDropTags = (call: unknown[]) => {
source: "api",
domain: "score",
projectId: PROJECT_ID,
+ sdkName: "langfuse-test",
+ sdkVersion: "0.0.0",
}),
);
};
@@ -116,7 +119,7 @@ const processScores = (
},
});
-describe("score validation drop metric (LFE-14345)", () => {
+describe("score validation drop metric", () => {
beforeEach(() => {
mocks.recordIncrement.mockClear();
mocks.validateAndInflateScoreOverride = undefined;