Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(worker): downsample high-volume per-observation warn logs (#17297)

Niklas Semmler1616e47e6c4b
diff --git a/worker/src/features/tokenisation/usage.ts b/worker/src/features/tokenisation/usage.ts
index a64a95d3c9..e35e329236 100644
--- a/worker/src/features/tokenisation/usage.ts
+++ b/worker/src/features/tokenisation/usage.ts
@@ -10,7 +10,7 @@ import {
} from "tiktoken";
import { z } from "zod";
-import { logger } from "@langfuse/shared/src/server";
+import { logger, recordIncrement } from "@langfuse/shared/src/server";
const OpenAiTokenConfig = z.object({
tokenizerModel: z.string().refine(isTiktokenModel, {
@@ -28,6 +28,14 @@ const OpenAiChatTokenConfig = z.object({
tokensPerName: z.number(),
});
+// An invalid tokenizer config is a static, per-model configuration problem, not
+// a per-event condition. Log once per model id so the signal survives without
+// one line per observation. tokenCount runs inside the token-count worker-thread
+// pool (each pool worker is its own module instance), so this dedup is per
+// worker-thread, not strictly per process — still a collapse from per-event to a
+// handful of lines per pod.
+const warnedInvalidTokenizerConfigModelIds = new Set<string>();
+
export function tokenCount(p: {
model: Model;
text: unknown;
@@ -63,11 +71,19 @@ type ChatMessage = {
function openAiTokenCount(p: { model: Model; text: unknown }) {
const config = OpenAiTokenConfig.safeParse(p.model.tokenizerConfig);
if (!config.success) {
- logger.warn(
- `Invalid tokenizer config for model ${p.model.id}: ${JSON.stringify(
- p.model.tokenizerConfig,
- )}, ${JSON.stringify(config.error)}`,
- );
+ // Counter fires per occurrence so the ongoing rate stays visible; the log
+ // line dedupes to once per model id (the config itself is static).
+ recordIncrement("langfuse.ingestion.tokenisation.invalid_config", 1, {
+ tokenizer: "openai",
+ });
+ if (!warnedInvalidTokenizerConfigModelIds.has(p.model.id)) {
+ warnedInvalidTokenizerConfigModelIds.add(p.model.id);
+ logger.warn(
+ `Invalid tokenizer config for model ${p.model.id}: ${JSON.stringify(
+ p.model.tokenizerConfig,
+ )}, ${JSON.stringify(config.error)}`,
+ );
+ }
return undefined;
}
@@ -84,11 +100,17 @@ function openAiTokenCount(p: { model: Model; text: unknown }) {
p.model.tokenizerConfig,
);
if (!parsedConfig.success) {
- logger.error(
- `Invalid tokenizer config for chat model ${
- p.model.id
- }: ${JSON.stringify(p.model.tokenizerConfig)}`,
- );
+ recordIncrement("langfuse.ingestion.tokenisation.invalid_config", 1, {
+ tokenizer: "openai_chat",
+ });
+ if (!warnedInvalidTokenizerConfigModelIds.has(p.model.id)) {
+ warnedInvalidTokenizerConfigModelIds.add(p.model.id);
+ logger.error(
+ `Invalid tokenizer config for chat model ${
+ p.model.id
+ }: ${JSON.stringify(p.model.tokenizerConfig)}`,
+ );
+ }
return undefined;
}
result = openAiChatTokenCount({
diff --git a/worker/src/queues/otelIngestionQueue.ts b/worker/src/queues/otelIngestionQueue.ts
index b4da0b5c16..ae71281f71 100644
--- a/worker/src/queues/otelIngestionQueue.ts
+++ b/worker/src/queues/otelIngestionQueue.ts
@@ -1,4 +1,5 @@
import { Job, Processor } from "bullmq";
+import { z } from "zod";
import {
clickhouseClient,
createIngestionEventSchema,
@@ -560,23 +561,40 @@ export const otelIngestionQueueProcessorBuilder = (
);
// We need to parse each incoming observation through our ingestion schema to make use of its included transformations.
const ingestionSchema = createIngestionEventSchema(isLangfuseInternal);
- const observations = events
- .filter((e) => getClickhouseEntityType(e.type) === "observation")
+ const candidateObservations = events.filter(
+ (e) => getClickhouseEntityType(e.type) === "observation",
+ );
+ let firstParseError: z.ZodError | undefined;
+ const observations = candidateObservations
.map((o) => ingestionSchema.safeParse(o))
.flatMap((o) => {
if (!o.success) {
- logger.warn(
- `Failed to parse otel observation for project ${projectId} in ${fileKey}: ${o.error}`,
- {
- error: o.error,
- fileKey,
- },
- );
+ firstParseError ??= o.error;
return [];
}
return [o.data];
});
+ // Per-observation parse failures are near-pure noise (customer-sent data
+ // that fails our schema). Aggregate to one metric + one sample warn per
+ // file instead of one line per observation.
+ const parseFailureCount =
+ candidateObservations.length - observations.length;
+ if (parseFailureCount > 0) {
+ recordIncrement(
+ "langfuse.ingestion.otel.observation_parse_failure",
+ parseFailureCount,
+ );
+ logger.warn(
+ `Failed to parse ${parseFailureCount}/${candidateObservations.length} otel observations for project ${projectId} in ${fileKey}: ${firstParseError}`,
+ {
+ error: firstParseError,
+ fileKey,
+ parseFailureCount,
+ },
+ );
+ }
+
// In the next row, we only consider observations. The traces will be recorded in processEventBatch.
recordIncrement("langfuse.ingestion.event", observations.length, {
source: "otel",
diff --git a/worker/src/services/IngestionService/index.ts b/worker/src/services/IngestionService/index.ts
index 00b3ab4484..01a44ecfdd 100644
--- a/worker/src/services/IngestionService/index.ts
+++ b/worker/src/services/IngestionService/index.ts
@@ -1644,7 +1644,10 @@ export class IngestionService {
return;
}
IngestionService.lastUsageTotalMismatchLogAt = now;
- logger.warn(
+ // The `langfuse.ingestion.usage_details.total_mismatch` metric above carries
+ // the aggregate signal; keep the detailed line at debug to avoid drowning
+ // warn-level log volume with a per-event customer-data condition.
+ logger.debug(
"Sum of provided non-total usage_details buckets exceeds provided total; the instrumentor may be sending an inclusive input alongside cache buckets",
{
projectId: observationRecord.project_id,
diff --git a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
index ab65d01760..3fe13e29b3 100644
--- a/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
+++ b/worker/src/services/IngestionService/tests/calculateTokenCost.unit.test.ts
@@ -1556,7 +1556,7 @@ describe("Token Cost Calculation", () => {
it("should warn when provided non-total buckets sum to more than the provided total", async () => {
(IngestionService as any).lastUsageTotalMismatchLogAt = 0;
- const warnSpy = vi.spyOn(logger, "warn");
+ const debugSpy = vi.spyOn(logger, "debug");
const generationId = uuidv4();
const eventRecord = await (mockIngestionService as any).createEventRecord(
@@ -1576,7 +1576,7 @@ describe("Token Cost Calculation", () => {
"testfile.txt",
);
- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>
+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>
String(message).includes("exceeds provided total"),
);
expect(mismatchWarnings).toHaveLength(1);
@@ -1599,7 +1599,7 @@ describe("Token Cost Calculation", () => {
it("should not warn when provided buckets are consistent with the provided total", async () => {
(IngestionService as any).lastUsageTotalMismatchLogAt = 0;
- const warnSpy = vi.spyOn(logger, "warn");
+ const debugSpy = vi.spyOn(logger, "debug");
const generationId = uuidv4();
await (mockIngestionService as any).createEventRecord(
@@ -1619,7 +1619,7 @@ describe("Token Cost Calculation", () => {
"testfile.txt",
);
- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>
+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>
String(message).includes("exceeds provided total"),
);
expect(mismatchWarnings).toHaveLength(0);
@@ -1627,7 +1627,7 @@ describe("Token Cost Calculation", () => {
it("should not warn when no total is provided", async () => {
(IngestionService as any).lastUsageTotalMismatchLogAt = 0;
- const warnSpy = vi.spyOn(logger, "warn");
+ const debugSpy = vi.spyOn(logger, "debug");
const generationId = uuidv4();
await (mockIngestionService as any).createEventRecord(
@@ -1645,7 +1645,7 @@ describe("Token Cost Calculation", () => {
"testfile.txt",
);
- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>
+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>
String(message).includes("exceeds provided total"),
);
expect(mismatchWarnings).toHaveLength(0);
@@ -1653,7 +1653,7 @@ describe("Token Cost Calculation", () => {
it("should warn on the direct event path even when no model is provided", async () => {
(IngestionService as any).lastUsageTotalMismatchLogAt = 0;
- const warnSpy = vi.spyOn(logger, "warn");
+ const debugSpy = vi.spyOn(logger, "debug");
const generationId = uuidv4();
await (mockIngestionService as any).createEventRecord(
@@ -1671,7 +1671,7 @@ describe("Token Cost Calculation", () => {
"testfile.txt",
);
- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>
+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>
String(message).includes("exceeds provided total"),
);
expect(mismatchWarnings).toHaveLength(1);
@@ -1683,7 +1683,7 @@ describe("Token Cost Calculation", () => {
it("should log the warning at most once per rate-limit interval", async () => {
(IngestionService as any).lastUsageTotalMismatchLogAt = 0;
- const warnSpy = vi.spyOn(logger, "warn");
+ const debugSpy = vi.spyOn(logger, "debug");
for (const spanId of [uuidv4(), uuidv4()]) {
await (mockIngestionService as any).createEventRecord(
@@ -1703,7 +1703,7 @@ describe("Token Cost Calculation", () => {
);
}
- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>
+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>
String(message).includes("exceeds provided total"),
);
expect(mismatchWarnings).toHaveLength(1);
@@ -1711,7 +1711,7 @@ describe("Token Cost Calculation", () => {
it("should warn on the legacy merge path when incoming events carry inconsistent usage", async () => {
(IngestionService as any).lastUsageTotalMismatchLogAt = 0;
- const warnSpy = vi.spyOn(logger, "warn");
+ const debugSpy = vi.spyOn(logger, "debug");
const generationId = uuidv4();
const events = [
@@ -1740,7 +1740,7 @@ describe("Token Cost Calculation", () => {
observationEventList: events,
});
- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>
+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>
String(message).includes("exceeds provided total"),
);
expect(mismatchWarnings).toHaveLength(1);
@@ -1753,7 +1753,7 @@ describe("Token Cost Calculation", () => {
it("should not warn on the legacy merge path when incoming events carry no usage", async () => {
(IngestionService as any).lastUsageTotalMismatchLogAt = 0;
- const warnSpy = vi.spyOn(logger, "warn");
+ const debugSpy = vi.spyOn(logger, "debug");
const generationId = uuidv4();
// Partial update without usage: the guard must not fire even if merged
@@ -1779,7 +1779,7 @@ describe("Token Cost Calculation", () => {
observationEventList: events,
});
- const mismatchWarnings = warnSpy.mock.calls.filter(([message]) =>
+ const mismatchWarnings = debugSpy.mock.calls.filter(([message]) =>
String(message).includes("exceeds provided total"),
);
expect(mismatchWarnings).toHaveLength(0);