🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(worker): tag queue failure metrics with a failure reason (#18397)
Niklas Semmler43dee3005e18
diff --git a/packages/shared/src/server/instrumentation/index.ts b/packages/shared/src/server/instrumentation/index.tsindex 6ebdd743f8..5617b5c5c1 100644--- a/packages/shared/src/server/instrumentation/index.ts+++ b/packages/shared/src/server/instrumentation/index.ts@@ -299,8 +299,12 @@ export const flushMetricsToCloudWatch = () => {};// Metrics ending with these suffixes have their tags flattened into the-// CloudWatch metric name (excluding "unit"). Other metrics are unaffected.+// CloudWatch metric name (excluding CW_UNFLATTENED_TAGS). Other metrics are+// unaffected.const CW_TAG_FLATTENED_SUFFIXES = [".depth", ".rate", ".dlq_oldest_age"];+// "reason" is a Datadog-only breakdown; flattening it would rename existing+// CloudWatch series.+const CW_UNFLATTENED_TAGS = new Set(["unit", "reason"]);function buildCloudWatchKey(stat: string,@@ -310,7 +314,7 @@ function buildCloudWatchKey(return stat;}const suffix = Object.entries(tags)- .filter(([k]) => k !== "unit")+ .filter(([k]) => !CW_UNFLATTENED_TAGS.has(k)).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}_${v}`).join(".");diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.tsindex 7b8f8d8e6b..8820b78f40 100644--- a/packages/shared/src/server/services/StorageService.ts+++ b/packages/shared/src/server/services/StorageService.ts@@ -102,8 +102,11 @@ function handleStorageError(err: unknown, operation: string): never {err.code === "EAI_AGAIN") {logger.error(`DNS lookup failure during ${operation}`, err);- throw new ServiceUnavailableError(- "Storage service temporarily unavailable due to network issues",+ throw Object.assign(+ new ServiceUnavailableError(+ "Storage service temporarily unavailable due to network issues",+ ),+ { cause: err },);}// For other errors, throw with the original cause preserveddiff --git a/worker/src/__tests__/workerManagerMetrics.test.ts b/worker/src/__tests__/workerManagerMetrics.test.tsindex ce67bfbb34..5df2f329f1 100644--- a/worker/src/__tests__/workerManagerMetrics.test.ts+++ b/worker/src/__tests__/workerManagerMetrics.test.ts@@ -130,7 +130,11 @@ describe("WorkerManager queue metrics", () => {mocks.handlers.get("stalled")?.("job-id");expect(mocks.recordIncrement.mock.calls).toEqual([- ["langfuse.queue.trace_delete.rate", 1, { type: "failed" }],+ [+ "langfuse.queue.trace_delete.rate",+ 1,+ { type: "failed", reason: "other" },+ ],["langfuse.queue.trace_delete.rate", 1, { type: "error" }],["langfuse.queue.trace_delete.rate", 1, { type: "stalled" }],]);@@ -168,9 +172,21 @@ describe("WorkerManager queue metrics", () => {);expect(mocks.recordIncrement.mock.calls).toEqual([- ["langfuse.queue.project_delete.rate", 1, { type: "failed" }],- ["langfuse.queue.project_delete.rate", 1, { type: "failed" }],- ["langfuse.queue.project_delete.rate", 1, { type: "failed_terminal" }],+ [+ "langfuse.queue.project_delete.rate",+ 1,+ { type: "failed", reason: "other" },+ ],+ [+ "langfuse.queue.project_delete.rate",+ 1,+ { type: "failed", reason: "other" },+ ],+ [+ "langfuse.queue.project_delete.rate",+ 1,+ { type: "failed_terminal", reason: "other" },+ ],]);});@@ -199,7 +215,7 @@ describe("WorkerManager queue metrics", () => {expect(mocks.recordIncrement).toHaveBeenCalledWith("langfuse.queue.score_delete.rate",1,- { type: "failed" },+ { type: "failed", reason: "s3_slowdown" },);mocks.handlers.get("failed")?.(diff --git a/worker/src/queues/__tests__/jobFailureReason.test.ts b/worker/src/queues/__tests__/jobFailureReason.test.tsnew file mode 100644index 0000000000..264a25488a--- /dev/null+++ b/worker/src/queues/__tests__/jobFailureReason.test.ts@@ -0,0 +1,112 @@+import { describe, expect, it } from "vitest";+import { z } from "zod";+import { classifyJobFailure } from "../jobFailureReason";++const wrapStorage = (cause: unknown) =>+ new Error("Failed to download file from S3", { cause });++const zodError = () => {+ const result = z.object({ id: z.string() }).safeParse({ id: 1 });+ if (result.success) throw new Error("expected zod failure");+ return result.error;+};++const jsonSyntaxError = () => {+ try {+ JSON.parse("{not json");+ } catch (e) {+ return e;+ }+ throw new Error("expected JSON.parse failure");+};++describe("classifyJobFailure", () => {+ it.each([+ [+ "wrapped S3 SlowDown",+ wrapStorage(+ Object.assign(new Error("Please reduce your request rate."), {+ name: "SlowDown",+ Code: "SlowDown",+ $metadata: { httpStatusCode: 503 },+ }),+ ),+ "s3_slowdown",+ ],+ [+ "wrapped S3 socket reset",+ wrapStorage(+ Object.assign(new Error("socket hang up"), {+ name: "TimeoutError",+ code: "ECONNRESET",+ }),+ ),+ "network",+ ],+ ["ClickHouse request timeout", new Error("Timeout error."), "network"],+ [+ "multi-address connect failure",+ wrapStorage(+ new AggregateError([+ Object.assign(new Error("connect ECONNREFUSED ::1:9000"), {+ code: "ECONNREFUSED",+ }),+ Object.assign(new Error("connect ETIMEDOUT 127.0.0.1:9000"), {+ code: "ETIMEDOUT",+ }),+ ]),+ ),+ "network",+ ],+ [+ "storage DNS failure",+ Object.assign(+ new Error(+ "Storage service temporarily unavailable due to network issues",+ ),+ {+ cause: Object.assign(new Error("getaddrinfo"), { code: "EAI_AGAIN" }),+ },+ ),+ "network",+ ],+ [+ "Postgres NUL byte",+ new Error(+ 'Invalid `prisma.$executeRaw()` invocation:\n\nRaw query failed. Code: `22021`. Message: `ERROR: invalid byte sequence for encoding "UTF8": 0x00`',+ ),+ "invalid_data",+ ],+ ["JSON.parse failure", jsonSyntaxError(), "invalid_data"],+ ["zod validation failure", zodError(), "invalid_data"],+ [+ "ClickHouse oversized JSON row",+ new Error(+ "Size of JSON object at position 104857600 is extremely large. Expected not greater than 10485760 bytes, but current is 104857605 bytes per row.",+ ),+ "payload_too_large",+ ],+ [+ "JS string length limit",+ new RangeError("Invalid string length"),+ "payload_too_large",+ ],+ [+ "Node max string size",+ Object.assign(+ new Error("Cannot create a string longer than 0x1fffffe8 characters"),+ { code: "ERR_STRING_TOO_LONG" },+ ),+ "payload_too_large",+ ],+ [+ "BullMQ stall limit",+ new Error("job stalled more than allowable limit"),+ "stalled",+ ],+ ["unknown error", new Error("Redis not available"), "other"],+ ["non-error value", "boom", "other"],+ ])("%s → %s", (_label, error, expected) => {+ expect(classifyJobFailure(error)).toBe(expected);+ });+});diff --git a/worker/src/queues/jobFailureReason.ts b/worker/src/queues/jobFailureReason.tsnew file mode 100644index 0000000000..7eedf290e7--- /dev/null+++ b/worker/src/queues/jobFailureReason.ts@@ -0,0 +1,95 @@+import { isS3SlowDownError } from "@langfuse/shared/src/server";++const MAX_CAUSE_DEPTH = 8;++export type JobFailureReason =+ | "s3_slowdown"+ | "payload_too_large"+ | "invalid_data"+ | "network"+ | "stalled"+ | "other";++/**+ * Low-cardinality failure category for queue metrics. Storage helpers wrap the+ * original error as `cause`, so the whole cause chain is inspected.+ */+export function classifyJobFailure(err: unknown): JobFailureReason {+ if (isS3SlowDownError(err)) return "s3_slowdown";++ const chain = causeChain(err);+ if (chain.some(isPayloadTooLarge)) return "payload_too_large";+ if (chain.some(isInvalidData)) return "invalid_data";+ if (chain.some(isNetworkError)) return "network";+ if (chain.some((e) => messageOf(e).includes("stalled more than allowable")))+ return "stalled";+ return "other";+}++/** Breadth-first over `cause` and `AggregateError.errors`. */+function causeChain(err: unknown): object[] {+ const chain: object[] = [];+ const pending: unknown[] = [err];+ while (pending.length > 0 && chain.length < MAX_CAUSE_DEPTH) {+ const current = pending.shift();+ if (!current || typeof current !== "object" || chain.includes(current)) {+ continue;+ }+ chain.push(current);+ pending.push((current as { cause?: unknown }).cause);+ const nested = (current as { errors?: unknown }).errors;+ if (Array.isArray(nested)) pending.push(...nested);+ }+ return chain;+}++function messageOf(err: object): string {+ const message = (err as { message?: unknown }).message;+ return typeof message === "string" ? message.toLowerCase() : "";+}++function codeOf(err: object): unknown {+ return (err as { code?: unknown }).code;+}++function isPayloadTooLarge(err: object): boolean {+ const message = messageOf(err);+ return (+ codeOf(err) === "ERR_STRING_TOO_LONG" ||+ message.includes("invalid string length") ||+ message.includes("cannot create a string longer than") ||+ (message.includes("size of json object") &&+ message.includes("extremely large"))+ );+}++function isInvalidData(err: object): boolean {+ const name = (err as { name?: unknown }).name;+ const message = messageOf(err);+ return (+ name === "SyntaxError" ||+ name === "ZodError" ||+ message.includes("invalid byte sequence") ||+ message.includes("cannot parse input")+ );+}++const NETWORK_ERROR_CODES = new Set([+ "ECONNRESET",+ "ECONNREFUSED",+ "ETIMEDOUT",+ "EPIPE",+ "EAI_AGAIN",+]);++function isNetworkError(err: object): boolean {+ const name = (err as { name?: unknown }).name;+ const message = messageOf(err);+ return (+ NETWORK_ERROR_CODES.has(codeOf(err) as string) ||+ name === "TimeoutError" ||+ message.includes("socket hang up") ||+ // @clickhouse/client request_timeout+ message.includes("timeout error")+ );+}diff --git a/worker/src/queues/workerManager.ts b/worker/src/queues/workerManager.tsindex b3d76bae48..4d583c7900 100644--- a/worker/src/queues/workerManager.ts+++ b/worker/src/queues/workerManager.ts@@ -17,6 +17,7 @@ import {import { WORKER_HOST_ID } from "../utils/hostId";import { logRetryableJobFailure } from "./jobFailureLog";import { SHARDED_QUEUE_BASE_NAMES } from "./shardedQueueRegistry";+import { classifyJobFailure } from "./jobFailureReason";export class WorkerManager {private static workers: { [key: string]: Worker } = {};@@ -179,8 +180,10 @@ export class WorkerManager {job,attemptsIncludeCurrentFailure: true,});+ const reason = classifyJobFailure(err);recordIncrement(baseMetric + ".rate", 1, {type: "failed",+ reason,...shardTag,});// BullMQ sets finishedOn only when it moves the job to the failed set@@ -188,6 +191,7 @@ export class WorkerManager {if (job?.finishedOn) {recordIncrement(baseMetric + ".rate", 1, {type: "failed_terminal",+ reason,...shardTag,});}diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.tsindex a2e6e701f5..299ad80588 100644--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts@@ -428,7 +428,7 @@ describe("ClickhouseWriter", () => {expect(serverExports.recordIncrement).toHaveBeenCalledWith("langfuse.queue.clickhouse_writer.rows_dropped",1,- { entity_type: TableName.EventsFull, format: "native" },+ { entity_type: TableName.EventsFull, format: "native", reason: "other" },);expect(logger.error).toHaveBeenCalledWith(expect.stringContaining("Max attempts reached"),@@ -508,7 +508,7 @@ describe("ClickhouseWriter", () => {expect(serverExports.recordIncrement).toHaveBeenCalledWith("langfuse.queue.clickhouse_writer.rows_dropped",1,- { entity_type: TableName.Traces, format: "json" },+ { entity_type: TableName.Traces, format: "json", reason: "other" },);});diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.tsindex d745b21357..99f58bdb7f 100644--- a/worker/src/services/ClickhouseWriter/index.ts+++ b/worker/src/services/ClickhouseWriter/index.ts@@ -22,6 +22,7 @@ import {type ClickhouseWriteStrategyFactory,} from "./writeStrategies";import { TableName, type RecordInsertType } from "./types";+import { classifyJobFailure } from "../../queues/jobFailureReason";export { TableName } from "./types";const MULTI_PROJECT_LOG_COMMENT_PROJECT_ID = "MULTI_PROJECT";@@ -472,6 +473,7 @@ export class ClickhouseWriter<);// Re-add the records to the queue with incremented attempts+ const reason = classifyJobFailure(err);let droppedCount = 0;queueItems.forEach((item) => {if (item.attempts < this.maxAttempts) {@@ -483,6 +485,7 @@ export class ClickhouseWriter<// TODO - Add to a dead letter queue in Redis rather than droppingrecordIncrement("langfuse.queue.clickhouse_writer.error", 1, {format: this.strategyFactory.format,+ reason,});droppedCount++;}@@ -492,7 +495,11 @@ export class ClickhouseWriter<recordIncrement("langfuse.queue.clickhouse_writer.rows_dropped",droppedCount,- { entity_type: tableName, format: this.strategyFactory.format },+ {+ entity_type: tableName,+ format: this.strategyFactory.format,+ reason,+ },);const droppedIds = queueItems