🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(worker): log project and failure state on queue job failures (#18437)
Niklas Semmlere82b2ad42b85
diff --git a/packages/shared/src/server/logger.test.ts b/packages/shared/src/server/logger.test.tsnew file mode 100644index 0000000000..09f285dde5--- /dev/null+++ b/packages/shared/src/server/logger.test.ts@@ -0,0 +1,67 @@+import { describe, expect, it, vi } from "vitest";++vi.mock("../env", () => ({+ env: {+ LANGFUSE_LOG_FORMAT: "json",+ LANGFUSE_LOG_LEVEL: "info",+ NODE_ENV: "test",+ },+}));++import { logger } from "./logger";++const jsonLine = (meta: Record<string, unknown>) => {+ const info = logger.format.transform({+ level: "error",+ message: "job failed",+ ...meta,+ });+ if (typeof info === "boolean") throw new Error("log line was filtered");+ return JSON.parse(info[Symbol.for("message")] as string);+};++describe("logger JSON format", () => {+ it("serialises a nested error and its cause chain", () => {+ const rootCause = new Error("socket hang up");+ const cause = Object.assign(+ new Error("Connection timed out", { cause: rootCause }),+ { name: "TimeoutError" },+ );+ const error = new Error("Failed to download file from S3", { cause });++ expect(jsonLine({ error, projectId: "project-1" })).toMatchObject({+ error: {+ message: "Failed to download file from S3",+ stack: error.stack,+ cause: {+ name: "TimeoutError",+ message: "Connection timed out",+ cause: { message: "socket hang up" },+ },+ },+ projectId: "project-1",+ });+ });++ it("serialises the same error in full on every line", () => {+ const error = new Error("NoSuchKey");++ jsonLine({ error });+ expect(jsonLine({ error })).toMatchObject({+ error: { message: "NoSuchKey" },+ });+ });++ it("keeps aggregated errors and survives a cyclic cause chain", () => {+ const error = new AggregateError([new Error("part failed")], "batch");+ error.cause = error;++ expect(jsonLine({ error })).toMatchObject({+ error: {+ message: "batch",+ cause: "[Circular]",+ errors: [{ message: "part failed" }],+ },+ });+ });+});diff --git a/packages/shared/src/server/logger.ts b/packages/shared/src/server/logger.tsindex edaac8d332..1979b274f0 100644--- a/packages/shared/src/server/logger.ts+++ b/packages/shared/src/server/logger.ts@@ -48,16 +48,15 @@ const severityFormat = function () {})();};-// Metadata is arbitrary caller-supplied data: `Error` serialises to `{}`, and-// circular or BigInt values make `JSON.stringify` throw. A logger that throws-// destroys the diagnostic it was called to emit, so degrade to a marker.-const stringifyMeta = (meta: Record<string, unknown>) => {- // Breaks cyclic `cause` chains: the replacer hands back a new object for- // every Error, so JSON.stringify's identity-based circular check never- // fires. An Error seen twice renders as "[Circular]" the second time.+// `Error` serialises to `{}`, so Errors anywhere in the metadata are expanded,+// and BigInt is stringified. Build one per log line: the cycle guard breaks+// cyclic `cause` chains — the replacer hands back a new object for every+// Error, so the stringifier's identity-based circular check never fires — and+// renders an Error seen twice in the same line as "[Circular]".+const createErrorReplacer = () => {const visitedErrors = new WeakSet<Error>();- const replacer = (_key: string, value: unknown) => {+ return (_key: string, value: unknown) => {if (value instanceof Error) {if (visitedErrors.has(value)) return "[Circular]";visitedErrors.add(value);@@ -73,14 +72,21 @@ const stringifyMeta = (meta: Record<string, unknown>) => {}return typeof value === "bigint" ? value.toString() : value;};+};+// Metadata is arbitrary caller-supplied data, and circular values make+// `JSON.stringify` throw. A logger that throws destroys the diagnostic it was+// called to emit, so degrade to a marker.+const stringifyMeta = (meta: Record<string, unknown>) => {try {- return JSON.stringify(meta, replacer);+ return JSON.stringify(meta, createErrorReplacer());} catch {return "[unserialisable log metadata]";}};+const jsonFormat = winston.format.json();+const getWinstonLogger = (nodeEnv: "development" | "production" | "test",minLevel = "info",@@ -105,7 +111,9 @@ const getWinstonLogger = (winston.format.timestamp(),tracingFormat(),severityFormat(),- winston.format.json(),+ winston.format((info) =>+ jsonFormat.transform(info, { replacer: createErrorReplacer() }),+ )(),);const format =diff --git a/worker/src/queues/workerManager.ts b/worker/src/queues/workerManager.tsindex 4d583c7900..c986b71f51 100644--- a/worker/src/queues/workerManager.ts+++ b/worker/src/queues/workerManager.ts@@ -23,16 +23,18 @@ export class WorkerManager {private static workers: { [key: string]: Worker } = {};private static extractProjectId(job: Job): string | undefined {- const data = job.data as {- payload?: {- projectId?: unknown;- authCheck?: { scope?: { projectId?: unknown } };- };- };+ const data = job.data as+ | {+ payload?: {+ projectId?: unknown;+ authCheck?: { scope?: { projectId?: unknown } };+ };+ }+ | undefined;const candidates = [- data.payload?.projectId,- data.payload?.authCheck?.scope?.projectId,+ data?.payload?.projectId,+ data?.payload?.authCheck?.scope?.projectId,];return candidates.find((candidate): candidate is string => {@@ -174,13 +176,20 @@ export class WorkerManager {// Add error handlingworker.on("failed", (job: Job | undefined, err: Error) => {+ const reason = classifyJobFailure(err);logRetryableJobFailure({message: `Queue job ${job?.name} with id ${job?.id} in ${queueName} failed`,error: err,job,attemptsIncludeCurrentFailure: true,+ fields: {+ projectId: job ? WorkerManager.extractProjectId(job) : undefined,+ jobId: job?.id,+ failureReason: reason,+ // true once BullMQ stops retrying: the job's events are dropped.+ terminal: Boolean(job?.finishedOn),+ },});- const reason = classifyJobFailure(err);recordIncrement(baseMetric + ".rate", 1, {type: "failed",reason,