🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(ingestion): tag metadata_dropped with projectId, drop OTel warn log (#16844)
Niklas Semmlera1b188ce5828
diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.tsindex 0ff5ac3b92..879cfc9cb4 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.metadataDropped.test.ts@@ -5,8 +5,8 @@* Spec under test:* - Counter name: langfuse.ingestion.metadata_dropped* - Tags: reason ∈ {non_object_top_level, parse_failure, primitive},- * source = otel, domain ∈ {trace, observation}- * - project_id must NOT be a metric tag (cardinality)+ * source = otel, domain ∈ {trace, observation}, projectId (low+ * cardinality: only projects emitting malformed metadata appear)* - No behavior change to what the processor returns* - Dotted-key metadata (langfuse.*.metadata.foo) stays increment-free*@@ -31,9 +31,7 @@ vi.mock("../instrumentation", async (importOriginal) => {// processToIngestionEvents awaits redis.set (seen-traces tracking); CI's// tests-shared job has REDIS_HOST set but no Redis server, so ioredis-// queues the command forever and the suite times out. Stub the client-// (not null — the null path adds a logger.warn, which would break the-// warn-cap exactly-10 assertion).+// queues the command forever and the suite times out. Stub the client.vi.mock("../redis/redis", async (importOriginal) => ({...(await importOriginal<typeof import("../redis/redis")>()),redis: { set: vi.fn().mockResolvedValue("OK") },@@ -112,9 +110,8 @@ const expectDropTags = (// recordIncrement(stat) defaults to 1; explicit 1 is equivalent.expect(value ?? 1).toBe(1);expect(tags).toEqual(expect.objectContaining(expected));- // Acceptance criterion: project_id is NOT a metric tag (cardinality).- expect(Object.keys(tags ?? {})).not.toContain("project_id");- expect(Object.keys(tags ?? {})).not.toContain("projectId");+ // projectId is a metric tag (low cardinality) for tenant attribution.+ expect(tags?.projectId).toBe(PROJECT_ID);};describe("OTel metadata_dropped metric (LFE-14342)", () => {@@ -299,8 +296,7 @@ describe("OTel metadata_dropped metric (LFE-14342)", () => {expect.objectContaining({ reason: expectedReason, source: "otel" }),);expect(["trace", "observation"]).toContain(tags?.domain);- expect(Object.keys(tags ?? {})).not.toContain("project_id");- expect(Object.keys(tags ?? {})).not.toContain("projectId");+ expect(tags?.projectId).toBe(PROJECT_ID);};it("counts a dropped attribute once when both pipelines run on one processor instance", async () => {@@ -458,8 +454,7 @@ describe("OTel metadata_dropped metric (LFE-14342)", () => {expect(["trace", "observation"]).toContain((tags as Record<string, string>)?.domain,);- expect(Object.keys(tags ?? {})).not.toContain("project_id");- expect(Object.keys(tags ?? {})).not.toContain("projectId");+ expect((tags as Record<string, string>)?.projectId).toBe(PROJECT_ID);}};@@ -613,41 +608,6 @@ describe("OTel metadata_dropped metric (LFE-14342)", () => {expectDropReasons(["parse_failure", "primitive"]);});});-- // RULING C: logger.warn from the drop path is capped at 10 per- // processor instance (per job); the metric keeps counting past the cap.- describe("warn cap", () => {- it("caps drop-path warns at 10 per instance while increments keep counting", async () => {- const warnSpy = vi- .spyOn(serverBarrel.logger, "warn")- .mockImplementation(() => serverBarrel.logger);-- try {- const spans = Array.from({ length: 12 }, (_, i) =>- makeSpan(- [- {- key: "langfuse.observation.metadata",- // Distinct malformed values so per-value dedup keeps all 12.- value: { stringValue: `{bad-${i}` },- },- ],- `00000000000000${(i + 1).toString(16).padStart(2, "0")}`,- ),- );-- const events = await createProcessor().processToIngestionEvents([- makeResourceSpan([], spans),- ]);-- expect(events.length).toBeGreaterThan(0);- expect(droppedCalls().length).toBeGreaterThan(10);- expect(warnSpy).toHaveBeenCalledTimes(10);- } finally {- warnSpy.mockRestore();- }- });- });});});diff --git a/packages/shared/src/server/otel/OtelIngestionProcessor.ts b/packages/shared/src/server/otel/OtelIngestionProcessor.tsindex 58d19b164a..7b96198f2a 100644--- a/packages/shared/src/server/otel/OtelIngestionProcessor.ts+++ b/packages/shared/src/server/otel/OtelIngestionProcessor.ts@@ -270,12 +270,10 @@ export class OtelIngestionProcessor {private static readonly MAX_OTEL_RECONSTRUCTED_ARRAY_SLOTS = 10_001;private static readonly MAX_OTEL_RECONSTRUCTED_PATH_DEPTH = 64;- private static readonly METADATA_DROP_WARN_CAP = 10;private static readonly ARRAY_ATTRIBUTE_DROP_WARN_CAP = 10;private seenTraces: Set<string> = new Set();private reportedMetadataDrops = new WeakMap<object, Set<string>>();- private metadataDropWarnCount = 0;private arrayAttributeDropWarnCounts = new Map<ReconstructedAttributeDropReason,number@@ -3274,7 +3272,6 @@ export class OtelIngestionProcessor {// pipelines) or the per-resourceSpan attributes object for resource// attributes — on (attribute key, reason). Distinct spans/resourceSpans// in one job count separately; the first-seen domain wins the tag.- // Warns are capped per instance, increments are not.private recordMetadataDropped(reason: string,context: MetadataDropContext,@@ -3295,18 +3292,8 @@ export class OtelIngestionProcessor {reason,source: "otel",domain,+ projectId: this.projectId,});- if (- this.metadataDropWarnCount < OtelIngestionProcessor.METADATA_DROP_WARN_CAP- ) {- this.metadataDropWarnCount += 1;- logger.warn("OTEL metadata attribute dropped", {- projectId: this.projectId,- reason,- domain,- attributeKey,- });- }}private parseMetadataAttribute(diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex 283563c8a8..3053af69b4 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -728,6 +728,7 @@ export class IngestionService {reason: "score_validation_dropped",source: "api",domain: "score",+ projectId,});}diff --git a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.tsindex e473afdfed..6d3334834d 100644--- a/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts+++ b/worker/src/services/IngestionService/tests/scoreValidationDroppedMetric.unit.test.ts@@ -10,8 +10,7 @@* events in the same batch are still written, unexpected errors still* reject the batch (and must NOT emit the drop counter — a rejected* batch is retried, not silently lost).- * - project_id is NOT a metric tag; tenant attribution via a log line- * that includes the projectId.+ * - projectId is a metric tag (low cardinality) for tenant attribution.*/import { expect, describe, it, vi, beforeEach } from "vitest";@@ -62,11 +61,9 @@ const expectScoreDropTags = (call: unknown[]) => {reason: "score_validation_dropped",source: "api",domain: "score",+ projectId: PROJECT_ID,}),);- // Acceptance criterion: project_id is NOT a metric tag (cardinality).- expect(Object.keys(tags ?? {})).not.toContain("project_id");- expect(Object.keys(tags ?? {})).not.toContain("projectId");};const createService = () => {@@ -146,8 +143,7 @@ describe("score validation drop metric (LFE-14345)", () => {expect(calls).toHaveLength(1);expectScoreDropTags(calls[0]);- // Acceptance criterion: tenant attribution via logs — a log line- // emitted during the drop includes the projectId.+ // The "no valid score records" log names the tenant.expect(JSON.stringify(loggerCalls)).toContain(PROJECT_ID);});