Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(worker): tag queue failure metrics with a failure reason (#18397)

Niklas Semmler43dee3005e18
diff --git a/packages/shared/src/server/instrumentation/index.ts b/packages/shared/src/server/instrumentation/index.ts
index 6ebdd743f8..5617b5c5c1 100644
--- a/packages/shared/src/server/instrumentation/index.ts
+++ b/packages/shared/src/server/instrumentation/index.ts
@@ -299,8 +299,12 @@ export const flushMetricsToCloudWatch = () => {
};
// Metrics ending with these suffixes have their tags flattened into the
-// CloudWatch metric name (excluding "unit"). Other metrics are unaffected.
+// CloudWatch metric name (excluding CW_UNFLATTENED_TAGS). Other metrics are
+// unaffected.
const CW_TAG_FLATTENED_SUFFIXES = [".depth", ".rate", ".dlq_oldest_age"];
+// "reason" is a Datadog-only breakdown; flattening it would rename existing
+// CloudWatch series.
+const CW_UNFLATTENED_TAGS = new Set(["unit", "reason"]);
function buildCloudWatchKey(
stat: string,
@@ -310,7 +314,7 @@ function buildCloudWatchKey(
return stat;
}
const suffix = Object.entries(tags)
- .filter(([k]) => k !== "unit")
+ .filter(([k]) => !CW_UNFLATTENED_TAGS.has(k))
.sort(([a], [b]) => a.localeCompare(b))
.map(([k, v]) => `${k}_${v}`)
.join(".");
diff --git a/packages/shared/src/server/services/StorageService.ts b/packages/shared/src/server/services/StorageService.ts
index 7b8f8d8e6b..8820b78f40 100644
--- a/packages/shared/src/server/services/StorageService.ts
+++ b/packages/shared/src/server/services/StorageService.ts
@@ -102,8 +102,11 @@ function handleStorageError(err: unknown, operation: string): never {
err.code === "EAI_AGAIN"
) {
logger.error(`DNS lookup failure during ${operation}`, err);
- throw new ServiceUnavailableError(
- "Storage service temporarily unavailable due to network issues",
+ throw Object.assign(
+ new ServiceUnavailableError(
+ "Storage service temporarily unavailable due to network issues",
+ ),
+ { cause: err },
);
}
// For other errors, throw with the original cause preserved
diff --git a/worker/src/__tests__/workerManagerMetrics.test.ts b/worker/src/__tests__/workerManagerMetrics.test.ts
index ce67bfbb34..5df2f329f1 100644
--- a/worker/src/__tests__/workerManagerMetrics.test.ts
+++ b/worker/src/__tests__/workerManagerMetrics.test.ts
@@ -130,7 +130,11 @@ describe("WorkerManager queue metrics", () => {
mocks.handlers.get("stalled")?.("job-id");
expect(mocks.recordIncrement.mock.calls).toEqual([
- ["langfuse.queue.trace_delete.rate", 1, { type: "failed" }],
+ [
+ "langfuse.queue.trace_delete.rate",
+ 1,
+ { type: "failed", reason: "other" },
+ ],
["langfuse.queue.trace_delete.rate", 1, { type: "error" }],
["langfuse.queue.trace_delete.rate", 1, { type: "stalled" }],
]);
@@ -168,9 +172,21 @@ describe("WorkerManager queue metrics", () => {
);
expect(mocks.recordIncrement.mock.calls).toEqual([
- ["langfuse.queue.project_delete.rate", 1, { type: "failed" }],
- ["langfuse.queue.project_delete.rate", 1, { type: "failed" }],
- ["langfuse.queue.project_delete.rate", 1, { type: "failed_terminal" }],
+ [
+ "langfuse.queue.project_delete.rate",
+ 1,
+ { type: "failed", reason: "other" },
+ ],
+ [
+ "langfuse.queue.project_delete.rate",
+ 1,
+ { type: "failed", reason: "other" },
+ ],
+ [
+ "langfuse.queue.project_delete.rate",
+ 1,
+ { type: "failed_terminal", reason: "other" },
+ ],
]);
});
@@ -199,7 +215,7 @@ describe("WorkerManager queue metrics", () => {
expect(mocks.recordIncrement).toHaveBeenCalledWith(
"langfuse.queue.score_delete.rate",
1,
- { type: "failed" },
+ { type: "failed", reason: "s3_slowdown" },
);
mocks.handlers.get("failed")?.(
diff --git a/worker/src/queues/__tests__/jobFailureReason.test.ts b/worker/src/queues/__tests__/jobFailureReason.test.ts
new file mode 100644
index 0000000000..264a25488a
--- /dev/null
+++ b/worker/src/queues/__tests__/jobFailureReason.test.ts
@@ -0,0 +1,112 @@
+import { describe, expect, it } from "vitest";
+import { z } from "zod";
+import { classifyJobFailure } from "../jobFailureReason";
+
+const wrapStorage = (cause: unknown) =>
+ new Error("Failed to download file from S3", { cause });
+
+const zodError = () => {
+ const result = z.object({ id: z.string() }).safeParse({ id: 1 });
+ if (result.success) throw new Error("expected zod failure");
+ return result.error;
+};
+
+const jsonSyntaxError = () => {
+ try {
+ JSON.parse("{not json");
+ } catch (e) {
+ return e;
+ }
+ throw new Error("expected JSON.parse failure");
+};
+
+describe("classifyJobFailure", () => {
+ it.each([
+ [
+ "wrapped S3 SlowDown",
+ wrapStorage(
+ Object.assign(new Error("Please reduce your request rate."), {
+ name: "SlowDown",
+ Code: "SlowDown",
+ $metadata: { httpStatusCode: 503 },
+ }),
+ ),
+ "s3_slowdown",
+ ],
+ [
+ "wrapped S3 socket reset",
+ wrapStorage(
+ Object.assign(new Error("socket hang up"), {
+ name: "TimeoutError",
+ code: "ECONNRESET",
+ }),
+ ),
+ "network",
+ ],
+ ["ClickHouse request timeout", new Error("Timeout error."), "network"],
+ [
+ "multi-address connect failure",
+ wrapStorage(
+ new AggregateError([
+ Object.assign(new Error("connect ECONNREFUSED ::1:9000"), {
+ code: "ECONNREFUSED",
+ }),
+ Object.assign(new Error("connect ETIMEDOUT 127.0.0.1:9000"), {
+ code: "ETIMEDOUT",
+ }),
+ ]),
+ ),
+ "network",
+ ],
+ [
+ "storage DNS failure",
+ Object.assign(
+ new Error(
+ "Storage service temporarily unavailable due to network issues",
+ ),
+ {
+ cause: Object.assign(new Error("getaddrinfo"), { code: "EAI_AGAIN" }),
+ },
+ ),
+ "network",
+ ],
+ [
+ "Postgres NUL byte",
+ new Error(
+ 'Invalid `prisma.$executeRaw()` invocation:\n\nRaw query failed. Code: `22021`. Message: `ERROR: invalid byte sequence for encoding "UTF8": 0x00`',
+ ),
+ "invalid_data",
+ ],
+ ["JSON.parse failure", jsonSyntaxError(), "invalid_data"],
+ ["zod validation failure", zodError(), "invalid_data"],
+ [
+ "ClickHouse oversized JSON row",
+ new Error(
+ "Size of JSON object at position 104857600 is extremely large. Expected not greater than 10485760 bytes, but current is 104857605 bytes per row.",
+ ),
+ "payload_too_large",
+ ],
+ [
+ "JS string length limit",
+ new RangeError("Invalid string length"),
+ "payload_too_large",
+ ],
+ [
+ "Node max string size",
+ Object.assign(
+ new Error("Cannot create a string longer than 0x1fffffe8 characters"),
+ { code: "ERR_STRING_TOO_LONG" },
+ ),
+ "payload_too_large",
+ ],
+ [
+ "BullMQ stall limit",
+ new Error("job stalled more than allowable limit"),
+ "stalled",
+ ],
+ ["unknown error", new Error("Redis not available"), "other"],
+ ["non-error value", "boom", "other"],
+ ])("%s → %s", (_label, error, expected) => {
+ expect(classifyJobFailure(error)).toBe(expected);
+ });
+});
diff --git a/worker/src/queues/jobFailureReason.ts b/worker/src/queues/jobFailureReason.ts
new file mode 100644
index 0000000000..7eedf290e7
--- /dev/null
+++ b/worker/src/queues/jobFailureReason.ts
@@ -0,0 +1,95 @@
+import { isS3SlowDownError } from "@langfuse/shared/src/server";
+
+const MAX_CAUSE_DEPTH = 8;
+
+export type JobFailureReason =
+ | "s3_slowdown"
+ | "payload_too_large"
+ | "invalid_data"
+ | "network"
+ | "stalled"
+ | "other";
+
+/**
+ * Low-cardinality failure category for queue metrics. Storage helpers wrap the
+ * original error as `cause`, so the whole cause chain is inspected.
+ */
+export function classifyJobFailure(err: unknown): JobFailureReason {
+ if (isS3SlowDownError(err)) return "s3_slowdown";
+
+ const chain = causeChain(err);
+ if (chain.some(isPayloadTooLarge)) return "payload_too_large";
+ if (chain.some(isInvalidData)) return "invalid_data";
+ if (chain.some(isNetworkError)) return "network";
+ if (chain.some((e) => messageOf(e).includes("stalled more than allowable")))
+ return "stalled";
+ return "other";
+}
+
+/** Breadth-first over `cause` and `AggregateError.errors`. */
+function causeChain(err: unknown): object[] {
+ const chain: object[] = [];
+ const pending: unknown[] = [err];
+ while (pending.length > 0 && chain.length < MAX_CAUSE_DEPTH) {
+ const current = pending.shift();
+ if (!current || typeof current !== "object" || chain.includes(current)) {
+ continue;
+ }
+ chain.push(current);
+ pending.push((current as { cause?: unknown }).cause);
+ const nested = (current as { errors?: unknown }).errors;
+ if (Array.isArray(nested)) pending.push(...nested);
+ }
+ return chain;
+}
+
+function messageOf(err: object): string {
+ const message = (err as { message?: unknown }).message;
+ return typeof message === "string" ? message.toLowerCase() : "";
+}
+
+function codeOf(err: object): unknown {
+ return (err as { code?: unknown }).code;
+}
+
+function isPayloadTooLarge(err: object): boolean {
+ const message = messageOf(err);
+ return (
+ codeOf(err) === "ERR_STRING_TOO_LONG" ||
+ message.includes("invalid string length") ||
+ message.includes("cannot create a string longer than") ||
+ (message.includes("size of json object") &&
+ message.includes("extremely large"))
+ );
+}
+
+function isInvalidData(err: object): boolean {
+ const name = (err as { name?: unknown }).name;
+ const message = messageOf(err);
+ return (
+ name === "SyntaxError" ||
+ name === "ZodError" ||
+ message.includes("invalid byte sequence") ||
+ message.includes("cannot parse input")
+ );
+}
+
+const NETWORK_ERROR_CODES = new Set([
+ "ECONNRESET",
+ "ECONNREFUSED",
+ "ETIMEDOUT",
+ "EPIPE",
+ "EAI_AGAIN",
+]);
+
+function isNetworkError(err: object): boolean {
+ const name = (err as { name?: unknown }).name;
+ const message = messageOf(err);
+ return (
+ NETWORK_ERROR_CODES.has(codeOf(err) as string) ||
+ name === "TimeoutError" ||
+ message.includes("socket hang up") ||
+ // @clickhouse/client request_timeout
+ message.includes("timeout error")
+ );
+}
diff --git a/worker/src/queues/workerManager.ts b/worker/src/queues/workerManager.ts
index b3d76bae48..4d583c7900 100644
--- a/worker/src/queues/workerManager.ts
+++ b/worker/src/queues/workerManager.ts
@@ -17,6 +17,7 @@ import {
import { WORKER_HOST_ID } from "../utils/hostId";
import { logRetryableJobFailure } from "./jobFailureLog";
import { SHARDED_QUEUE_BASE_NAMES } from "./shardedQueueRegistry";
+import { classifyJobFailure } from "./jobFailureReason";
export class WorkerManager {
private static workers: { [key: string]: Worker } = {};
@@ -179,8 +180,10 @@ export class WorkerManager {
job,
attemptsIncludeCurrentFailure: true,
});
+ const reason = classifyJobFailure(err);
recordIncrement(baseMetric + ".rate", 1, {
type: "failed",
+ reason,
...shardTag,
});
// BullMQ sets finishedOn only when it moves the job to the failed set
@@ -188,6 +191,7 @@ export class WorkerManager {
if (job?.finishedOn) {
recordIncrement(baseMetric + ".rate", 1, {
type: "failed_terminal",
+ reason,
...shardTag,
});
}
diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
index a2e6e701f5..299ad80588 100644
--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
@@ -428,7 +428,7 @@ describe("ClickhouseWriter", () => {
expect(serverExports.recordIncrement).toHaveBeenCalledWith(
"langfuse.queue.clickhouse_writer.rows_dropped",
1,
- { entity_type: TableName.EventsFull, format: "native" },
+ { entity_type: TableName.EventsFull, format: "native", reason: "other" },
);
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining("Max attempts reached"),
@@ -508,7 +508,7 @@ describe("ClickhouseWriter", () => {
expect(serverExports.recordIncrement).toHaveBeenCalledWith(
"langfuse.queue.clickhouse_writer.rows_dropped",
1,
- { entity_type: TableName.Traces, format: "json" },
+ { entity_type: TableName.Traces, format: "json", reason: "other" },
);
});
diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.ts
index d745b21357..99f58bdb7f 100644
--- a/worker/src/services/ClickhouseWriter/index.ts
+++ b/worker/src/services/ClickhouseWriter/index.ts
@@ -22,6 +22,7 @@ import {
type ClickhouseWriteStrategyFactory,
} from "./writeStrategies";
import { TableName, type RecordInsertType } from "./types";
+import { classifyJobFailure } from "../../queues/jobFailureReason";
export { TableName } from "./types";
const MULTI_PROJECT_LOG_COMMENT_PROJECT_ID = "MULTI_PROJECT";
@@ -472,6 +473,7 @@ export class ClickhouseWriter<
);
// Re-add the records to the queue with incremented attempts
+ const reason = classifyJobFailure(err);
let droppedCount = 0;
queueItems.forEach((item) => {
if (item.attempts < this.maxAttempts) {
@@ -483,6 +485,7 @@ export class ClickhouseWriter<
// TODO - Add to a dead letter queue in Redis rather than dropping
recordIncrement("langfuse.queue.clickhouse_writer.error", 1, {
format: this.strategyFactory.format,
+ reason,
});
droppedCount++;
}
@@ -492,7 +495,11 @@ export class ClickhouseWriter<
recordIncrement(
"langfuse.queue.clickhouse_writer.rows_dropped",
droppedCount,
- { entity_type: tableName, format: this.strategyFactory.format },
+ {
+ entity_type: tableName,
+ format: this.strategyFactory.format,
+ reason,
+ },
);
const droppedIds = queueItems