🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(worker): downsample high-volume per-observation warn logs (#17297)
Niklas Semmler1616e47e6c4b
diff --git a/worker/src/features/tokenisation/usage.ts b/worker/src/features/tokenisation/usage.tsindex a64a95d3c9..e35e329236 100644--- a/worker/src/features/tokenisation/usage.ts+++ b/worker/src/features/tokenisation/usage.ts@@ -10,7 +10,7 @@ import {} from "tiktoken";import { z } from "zod";-import { logger } from "@langfuse/shared/src/server";+import { logger, recordIncrement } from "@langfuse/shared/src/server";const OpenAiTokenConfig = z.object({tokenizerModel: z.string().refine(isTiktokenModel, {@@ -28,6 +28,14 @@ const OpenAiChatTokenConfig = z.object({tokensPerName: z.number(),});+// An invalid tokenizer config is a static, per-model configuration problem, not+// a per-event condition. Log once per model id so the signal survives without+// one line per observation. tokenCount runs inside the token-count worker-thread+// pool (each pool worker is its own module instance), so this dedup is per+// worker-thread, not strictly per process — still a collapse from per-event to a+// handful of lines per pod.+const warnedInvalidTokenizerConfigModelIds = new Set<string>();+export function tokenCount(p: {model: Model;text: unknown;@@ -63,11 +71,19 @@ type ChatMessage = {function openAiTokenCount(p: { model: Model; text: unknown }) {const config = OpenAiTokenConfig.safeParse(p.model.tokenizerConfig);if (!config.success) {- logger.warn(- `Invalid tokenizer config for model ${p.model.id}: ${JSON.stringify(- p.model.tokenizerConfig,- )}, ${JSON.stringify(config.error)}`,- );+ // Counter fires per occurrence so the ongoing rate stays visible; the log+ // line dedupes to once per model id (the config itself is static).+ recordIncrement("langfuse.ingestion.tokenisation.invalid_config", 1, {+ tokenizer: "openai",+ });+ if (!warnedInvalidTokenizerConfigModelIds.has(p.model.id)) {+ warnedInvalidTokenizerConfigModelIds.add(p.model.id);+ logger.warn(+ `Invalid tokenizer config for model ${p.model.id}: ${JSON.stringify(+ p.model.tokenizerConfig,+ )}, ${JSON.stringify(config.error)}`,+ );+ }return undefined;}@@ -84,11 +100,17 @@ function openAiTokenCount(p: { model: Model; text: unknown }) {p.model.tokenizerConfig,);if (!parsedConfig.success) {- logger.error(- `Invalid tokenizer config for chat model ${- p.model.id- }: ${JSON.stringify(p.model.tokenizerConfig)}`,- );+ recordIncrement("langfuse.ingestion.tokenisation.invalid_config", 1, {+ tokenizer: "openai_chat",+ });+ if (!warnedInvalidTokenizerConfigModelIds.has(p.model.id)) {+ warnedInvalidTokenizerConfigModelIds.add(p.model.id);+ logger.error(+ `Invalid tokenizer config for chat model ${+ p.model.id+ }: ${JSON.stringify(p.model.tokenizerConfig)}`,+ );+ }return undefined;}result = openAiChatTokenCount({diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.tsindex b4da0b5c16..ae71281f71 100644--- a/worker/src/queues/otelIngestionQueue.ts+++ b/worker/src/queues/otelIngestionQueue.ts@@ -1,4 +1,5 @@import { Job, Processor } from "bullmq";+import { z } from "zod";import {clickhouseClient,createIngestionEventSchema,@@ -560,23 +561,40 @@ export const otelIngestionQueueProcessorBuilder = ();// We need to parse each incoming observation through our ingestion schema to make use of its included transformations.const ingestionSchema = createIngestionEventSchema(isLangfuseInternal);- const observations = events- .filter((e) => getClickhouseEntityType(e.type) === "observation")+ const candidateObservations = events.filter(+ (e) => getClickhouseEntityType(e.type) === "observation",+ );+ let firstParseError: z.ZodError | undefined;+ const observations = candidateObservations.map((o) => ingestionSchema.safeParse(o)).flatMap((o) => {if (!o.success) {- logger.warn(- `Failed to parse otel observation for project ${projectId} in ${fileKey}: ${o.error}`,- {- error: o.error,- fileKey,- },- );+ firstParseError ??= o.error;return [];}return [o.data];});+ // Per-observation parse failures are near-pure noise (customer-sent data+ // that fails our schema). Aggregate to one metric + one sample warn per+ // file instead of one line per observation.+ const parseFailureCount =+ candidateObservations.length - observations.length;+ if (parseFailureCount > 0) {+ recordIncrement(+ "langfuse.ingestion.otel.observation_parse_failure",+ parseFailureCount,+ );+ logger.warn(+ `Failed to parse ${parseFailureCount}/${candidateObservations.length} otel observations for project ${projectId} in ${fileKey}: ${firstParseError}`,+ {+ error: firstParseError,+ fileKey,+ parseFailureCount,+ },+ );+ }+// In the next row, we only consider observations. The traces will be recorded in processEventBatch.recordIncrement("langfuse.ingestion.event", observations.length, {source: "otel",diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.tsindex 00b3ab4484..01a44ecfdd 100644--- a/worker/src/services/IngestionService/index.ts+++ b/worker/src/services/IngestionService/index.ts@@ -1644,7 +1644,10 @@ export class IngestionService {return;}IngestionService.lastUsageTotalMismatchLogAt = now;- logger.warn(+ // The `langfuse.ingestion.usage_details.total_mismatch` metric above carries+ // the aggregate signal; keep the detailed line at debug to avoid drowning+ // warn-level log volume with a per-event customer-data condition.+ logger.debug("Sum of provided non-total usage_details buckets exceeds provided total; the instrumentor may be sending an inclusive input alongside cache buckets",{projectId: observationRecord.project_id,diff --git a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.tsindex ab65d01760..3fe13e29b3 100644--- a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts+++ b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts@@ -1556,7 +1556,7 @@ describe("Token Cost Calculation", () => {it("should warn when provided non-total buckets sum to more than the provided total", async () => {(IngestionService as any).lastUsageTotalMismatchLogAt = 0;- const warnSpy = vi.spyOn(logger, "warn");+ const debugSpy = vi.spyOn(logger, "debug");const generationId = uuidv4();const eventRecord = await (mockIngestionService as any).createEventRecord(@@ -1576,7 +1576,7 @@ describe("Token Cost Calculation", () => {"testfile.txt",);- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>String(message).includes("exceeds provided total"),);expect(mismatchWarnings).toHaveLength(1);@@ -1599,7 +1599,7 @@ describe("Token Cost Calculation", () => {it("should not warn when provided buckets are consistent with the provided total", async () => {(IngestionService as any).lastUsageTotalMismatchLogAt = 0;- const warnSpy = vi.spyOn(logger, "warn");+ const debugSpy = vi.spyOn(logger, "debug");const generationId = uuidv4();await (mockIngestionService as any).createEventRecord(@@ -1619,7 +1619,7 @@ describe("Token Cost Calculation", () => {"testfile.txt",);- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>String(message).includes("exceeds provided total"),);expect(mismatchWarnings).toHaveLength(0);@@ -1627,7 +1627,7 @@ describe("Token Cost Calculation", () => {it("should not warn when no total is provided", async () => {(IngestionService as any).lastUsageTotalMismatchLogAt = 0;- const warnSpy = vi.spyOn(logger, "warn");+ const debugSpy = vi.spyOn(logger, "debug");const generationId = uuidv4();await (mockIngestionService as any).createEventRecord(@@ -1645,7 +1645,7 @@ describe("Token Cost Calculation", () => {"testfile.txt",);- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>String(message).includes("exceeds provided total"),);expect(mismatchWarnings).toHaveLength(0);@@ -1653,7 +1653,7 @@ describe("Token Cost Calculation", () => {it("should warn on the direct event path even when no model is provided", async () => {(IngestionService as any).lastUsageTotalMismatchLogAt = 0;- const warnSpy = vi.spyOn(logger, "warn");+ const debugSpy = vi.spyOn(logger, "debug");const generationId = uuidv4();await (mockIngestionService as any).createEventRecord(@@ -1671,7 +1671,7 @@ describe("Token Cost Calculation", () => {"testfile.txt",);- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>String(message).includes("exceeds provided total"),);expect(mismatchWarnings).toHaveLength(1);@@ -1683,7 +1683,7 @@ describe("Token Cost Calculation", () => {it("should log the warning at most once per rate-limit interval", async () => {(IngestionService as any).lastUsageTotalMismatchLogAt = 0;- const warnSpy = vi.spyOn(logger, "warn");+ const debugSpy = vi.spyOn(logger, "debug");for (const spanId of [uuidv4(), uuidv4()]) {await (mockIngestionService as any).createEventRecord(@@ -1703,7 +1703,7 @@ describe("Token Cost Calculation", () => {);}- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>String(message).includes("exceeds provided total"),);expect(mismatchWarnings).toHaveLength(1);@@ -1711,7 +1711,7 @@ describe("Token Cost Calculation", () => {it("should warn on the legacy merge path when incoming events carry inconsistent usage", async () => {(IngestionService as any).lastUsageTotalMismatchLogAt = 0;- const warnSpy = vi.spyOn(logger, "warn");+ const debugSpy = vi.spyOn(logger, "debug");const generationId = uuidv4();const events = [@@ -1740,7 +1740,7 @@ describe("Token Cost Calculation", () => {observationEventList: events,});- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>String(message).includes("exceeds provided total"),);expect(mismatchWarnings).toHaveLength(1);@@ -1753,7 +1753,7 @@ describe("Token Cost Calculation", () => {it("should not warn on the legacy merge path when incoming events carry no usage", async () => {(IngestionService as any).lastUsageTotalMismatchLogAt = 0;- const warnSpy = vi.spyOn(logger, "warn");+ const debugSpy = vi.spyOn(logger, "debug");const generationId = uuidv4();// Partial update without usage: the guard must not fire even if merged@@ -1779,7 +1779,7 @@ describe("Token Cost Calculation", () => {observationEventList: events,});- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>String(message).includes("exceeds provided total"),);expect(mismatchWarnings).toHaveLength(0);