Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(worker): log project and failure state on queue job failures (#18437)

Niklas Semmlere82b2ad42b85
diff --git a/packages/shared/src/server/logger.test.ts b/packages/shared/src/server/logger.test.ts
new file mode 100644
index 0000000000..09f285dde5
--- /dev/null
+++ b/packages/shared/src/server/logger.test.ts
@@ -0,0 +1,67 @@
+import { describe, expect, it, vi } from "vitest";
+
+vi.mock("../env", () => ({
+ env: {
+ LANGFUSE_LOG_FORMAT: "json",
+ LANGFUSE_LOG_LEVEL: "info",
+ NODE_ENV: "test",
+ },
+}));
+
+import { logger } from "./logger";
+
+const jsonLine = (meta: Record<string, unknown>) => {
+ const info = logger.format.transform({
+ level: "error",
+ message: "job failed",
+ ...meta,
+ });
+ if (typeof info === "boolean") throw new Error("log line was filtered");
+ return JSON.parse(info[Symbol.for("message")] as string);
+};
+
+describe("logger JSON format", () => {
+ it("serialises a nested error and its cause chain", () => {
+ const rootCause = new Error("socket hang up");
+ const cause = Object.assign(
+ new Error("Connection timed out", { cause: rootCause }),
+ { name: "TimeoutError" },
+ );
+ const error = new Error("Failed to download file from S3", { cause });
+
+ expect(jsonLine({ error, projectId: "project-1" })).toMatchObject({
+ error: {
+ message: "Failed to download file from S3",
+ stack: error.stack,
+ cause: {
+ name: "TimeoutError",
+ message: "Connection timed out",
+ cause: { message: "socket hang up" },
+ },
+ },
+ projectId: "project-1",
+ });
+ });
+
+ it("serialises the same error in full on every line", () => {
+ const error = new Error("NoSuchKey");
+
+ jsonLine({ error });
+ expect(jsonLine({ error })).toMatchObject({
+ error: { message: "NoSuchKey" },
+ });
+ });
+
+ it("keeps aggregated errors and survives a cyclic cause chain", () => {
+ const error = new AggregateError([new Error("part failed")], "batch");
+ error.cause = error;
+
+ expect(jsonLine({ error })).toMatchObject({
+ error: {
+ message: "batch",
+ cause: "[Circular]",
+ errors: [{ message: "part failed" }],
+ },
+ });
+ });
+});
diff --git a/packages/shared/src/server/logger.ts b/packages/shared/src/server/logger.ts
index edaac8d332..1979b274f0 100644
--- a/packages/shared/src/server/logger.ts
+++ b/packages/shared/src/server/logger.ts
@@ -48,16 +48,15 @@ const severityFormat = function () {
})();
};
-// Metadata is arbitrary caller-supplied data: `Error` serialises to `{}`, and
-// circular or BigInt values make `JSON.stringify` throw. A logger that throws
-// destroys the diagnostic it was called to emit, so degrade to a marker.
-const stringifyMeta = (meta: Record<string, unknown>) => {
- // Breaks cyclic `cause` chains: the replacer hands back a new object for
- // every Error, so JSON.stringify's identity-based circular check never
- // fires. An Error seen twice renders as "[Circular]" the second time.
+// `Error` serialises to `{}`, so Errors anywhere in the metadata are expanded,
+// and BigInt is stringified. Build one per log line: the cycle guard breaks
+// cyclic `cause` chains — the replacer hands back a new object for every
+// Error, so the stringifier's identity-based circular check never fires — and
+// renders an Error seen twice in the same line as "[Circular]".
+const createErrorReplacer = () => {
const visitedErrors = new WeakSet<Error>();
- const replacer = (_key: string, value: unknown) => {
+ return (_key: string, value: unknown) => {
if (value instanceof Error) {
if (visitedErrors.has(value)) return "[Circular]";
visitedErrors.add(value);
@@ -73,14 +72,21 @@ const stringifyMeta = (meta: Record<string, unknown>) => {
}
return typeof value === "bigint" ? value.toString() : value;
};
+};
+// Metadata is arbitrary caller-supplied data, and circular values make
+// `JSON.stringify` throw. A logger that throws destroys the diagnostic it was
+// called to emit, so degrade to a marker.
+const stringifyMeta = (meta: Record<string, unknown>) => {
try {
- return JSON.stringify(meta, replacer);
+ return JSON.stringify(meta, createErrorReplacer());
} catch {
return "[unserialisable log metadata]";
}
};
+const jsonFormat = winston.format.json();
+
const getWinstonLogger = (
nodeEnv: "development" | "production" | "test",
minLevel = "info",
@@ -105,7 +111,9 @@ const getWinstonLogger = (
winston.format.timestamp(),
tracingFormat(),
severityFormat(),
- winston.format.json(),
+ winston.format((info) =>
+ jsonFormat.transform(info, { replacer: createErrorReplacer() }),
+ )(),
);
const format =
diff --git a/worker/src/queues/workerManager.ts b/worker/src/queues/workerManager.ts
index 4d583c7900..c986b71f51 100644
--- a/worker/src/queues/workerManager.ts
+++ b/worker/src/queues/workerManager.ts
@@ -23,16 +23,18 @@ export class WorkerManager {
private static workers: { [key: string]: Worker } = {};
private static extractProjectId(job: Job): string | undefined {
- const data = job.data as {
- payload?: {
- projectId?: unknown;
- authCheck?: { scope?: { projectId?: unknown } };
- };
- };
+ const data = job.data as
+ | {
+ payload?: {
+ projectId?: unknown;
+ authCheck?: { scope?: { projectId?: unknown } };
+ };
+ }
+ | undefined;
const candidates = [
- data.payload?.projectId,
- data.payload?.authCheck?.scope?.projectId,
+ data?.payload?.projectId,
+ data?.payload?.authCheck?.scope?.projectId,
];
return candidates.find((candidate): candidate is string => {
@@ -174,13 +176,20 @@ export class WorkerManager {
// Add error handling
worker.on("failed", (job: Job | undefined, err: Error) => {
+ const reason = classifyJobFailure(err);
logRetryableJobFailure({
message: `Queue job ${job?.name} with id ${job?.id} in ${queueName} failed`,
error: err,
job,
attemptsIncludeCurrentFailure: true,
+ fields: {
+ projectId: job ? WorkerManager.extractProjectId(job) : undefined,
+ jobId: job?.id,
+ failureReason: reason,
+ // true once BullMQ stops retrying: the job's events are dropped.
+ terminal: Boolean(job?.finishedOn),
+ },
});
- const reason = classifyJobFailure(err);
recordIncrement(baseMetric + ".rate", 1, {
type: "failed",
reason,