Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(ingestion): tag metadata_dropped with projectId, drop OTel warn log (#16844)

Niklas Semmlera1b188ce5828
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
index 0ff5ac3b92..879cfc9cb4 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts
@@ -5,8 +5,8 @@
* Spec under test:
* - Counter name: langfuse.ingestion.metadata_dropped
* - Tags: reason ∈ {non_object_top_level, parse_failure, primitive},
- * source = otel, domain ∈ {trace, observation}
- * - project_id must NOT be a metric tag (cardinality)
+ * source = otel, domain ∈ {trace, observation}, projectId (low
+ * cardinality: only projects emitting malformed metadata appear)
* - No behavior change to what the processor returns
* - Dotted-key metadata (langfuse.*.metadata.foo) stays increment-free
*
@@ -31,9 +31,7 @@ vi.mock("../instrumentation", async (importOriginal) => {
// processToIngestionEvents awaits redis.set (seen-traces tracking); CI's
// tests-shared job has REDIS_HOST set but no Redis server, so ioredis
-// queues the command forever and the suite times out. Stub the client
-// (not null — the null path adds a logger.warn, which would break the
-// warn-cap exactly-10 assertion).
+// queues the command forever and the suite times out. Stub the client.
vi.mock("../redis/redis", async (importOriginal) => ({
...(await importOriginal<typeof import("../redis/redis")>()),
redis: { set: vi.fn().mockResolvedValue("OK") },
@@ -112,9 +110,8 @@ const expectDropTags = (
// recordIncrement(stat) defaults to 1; explicit 1 is equivalent.
expect(value ?? 1).toBe(1);
expect(tags).toEqual(expect.objectContaining(expected));
- // Acceptance criterion: project_id is NOT a metric tag (cardinality).
- expect(Object.keys(tags ?? {})).not.toContain("project_id");
- expect(Object.keys(tags ?? {})).not.toContain("projectId");
+ // projectId is a metric tag (low cardinality) for tenant attribution.
+ expect(tags?.projectId).toBe(PROJECT_ID);
};
describe("OTel metadata_dropped metric (LFE-14342)", () => {
@@ -299,8 +296,7 @@ describe("OTel metadata_dropped metric (LFE-14342)", () => {
expect.objectContaining({ reason: expectedReason, source: "otel" }),
);
expect(["trace", "observation"]).toContain(tags?.domain);
- expect(Object.keys(tags ?? {})).not.toContain("project_id");
- expect(Object.keys(tags ?? {})).not.toContain("projectId");
+ expect(tags?.projectId).toBe(PROJECT_ID);
};
it("counts a dropped attribute once when both pipelines run on one processor instance", async () => {
@@ -458,8 +454,7 @@ describe("OTel metadata_dropped metric (LFE-14342)", () => {
expect(["trace", "observation"]).toContain(
(tags as Record<string, string>)?.domain,
);
- expect(Object.keys(tags ?? {})).not.toContain("project_id");
- expect(Object.keys(tags ?? {})).not.toContain("projectId");
+ expect((tags as Record<string, string>)?.projectId).toBe(PROJECT_ID);
}
};
@@ -613,41 +608,6 @@ describe("OTel metadata_dropped metric (LFE-14342)", () => {
expectDropReasons(["parse_failure", "primitive"]);
});
});
-
- // RULING C: logger.warn from the drop path is capped at 10 per
- // processor instance (per job); the metric keeps counting past the cap.
- describe("warn cap", () => {
- it("caps drop-path warns at 10 per instance while increments keep counting", async () => {
- const warnSpy = vi
- .spyOn(serverBarrel.logger, "warn")
- .mockImplementation(() => serverBarrel.logger);
-
- try {
- const spans = Array.from({ length: 12 }, (_, i) =>
- makeSpan(
- [
- {
- key: "langfuse.observation.metadata",
- // Distinct malformed values so per-value dedup keeps all 12.
- value: { stringValue: `{bad-${i}` },
- },
- ],
- `00000000000000${(i + 1).toString(16).padStart(2, "0")}`,
- ),
- );
-
- const events = await createProcessor().processToIngestionEvents([
- makeResourceSpan([], spans),
- ]);
-
- expect(events.length).toBeGreaterThan(0);
- expect(droppedCalls().length).toBeGreaterThan(10);
- expect(warnSpy).toHaveBeenCalledTimes(10);
- } finally {
- warnSpy.mockRestore();
- }
- });
- });
});
});
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
index 58d19b164a..7b96198f2a 100644
--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts
+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts
@@ -270,12 +270,10 @@ export class OtelIngestionProcessor {
private static readonly MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS = 10_001;
private static readonly MAX_OTEL_RECONSTRUCTED_PATH_DEPTH = 64;
- private static readonly METADATA_DROP_WARN_CAP = 10;
private static readonly ARRAY_ATTRIBUTE_DROP_WARN_CAP = 10;
private seenTraces: Set<string> = new Set();
private reportedMetadataDrops = new WeakMap<object, Set<string>>();
- private metadataDropWarnCount = 0;
private arrayAttributeDropWarnCounts = new Map<
ReconstructedAttributeDropReason,
number
@@ -3274,7 +3272,6 @@ export class OtelIngestionProcessor {
// pipelines) or the per-resourceSpan attributes object for resource
// attributes — on (attribute key, reason). Distinct spans/resourceSpans
// in one job count separately; the first-seen domain wins the tag.
- // Warns are capped per instance, increments are not.
private recordMetadataDropped(
reason: string,
context: MetadataDropContext,
@@ -3295,18 +3292,8 @@ export class OtelIngestionProcessor {
reason,
source: "otel",
domain,
+ projectId: this.projectId,
});
- if (
- this.metadataDropWarnCount < OtelIngestionProcessor.METADATA_DROP_WARN_CAP
- ) {
- this.metadataDropWarnCount += 1;
- logger.warn("OTEL metadata attribute dropped", {
- projectId: this.projectId,
- reason,
- domain,
- attributeKey,
- });
- }
}
private parseMetadataAttribute(
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 283563c8a8..3053af69b4 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -728,6 +728,7 @@ export class IngestionService {
reason: "score_validation_dropped",
source: "api",
domain: "score",
+ projectId,
});
}
diff --git a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts
index e473afdfed..6d3334834d 100644
--- a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts
@@ -10,8 +10,7 @@
* events in the same batch are still written, unexpected errors still
* reject the batch (and must NOT emit the drop counter — a rejected
* batch is retried, not silently lost).
- * - project_id is NOT a metric tag; tenant attribution via a log line
- * that includes the projectId.
+ * - projectId is a metric tag (low cardinality) for tenant attribution.
*/
import { expect, describe, it, vi, beforeEach } from "vitest";
@@ -62,11 +61,9 @@ const expectScoreDropTags = (call: unknown[]) => {
reason: "score_validation_dropped",
source: "api",
domain: "score",
+ projectId: PROJECT_ID,
}),
);
- // Acceptance criterion: project_id is NOT a metric tag (cardinality).
- expect(Object.keys(tags ?? {})).not.toContain("project_id");
- expect(Object.keys(tags ?? {})).not.toContain("projectId");
};
const createService = () => {
@@ -146,8 +143,7 @@ describe("score validation drop metric (LFE-14345)", () => {
expect(calls).toHaveLength(1);
expectScoreDropTags(calls[0]);
- // Acceptance criterion: tenant attribution via logs — a log line
- // emitted during the drop includes the projectId.
+ // The "no valid score records" log names the tenant.
expect(JSON.stringify(loggerCalls)).toContain(PROJECT_ID);
});