🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): stop counting earlier attempts as queue wait time (#19039)
Niklas Semmlerf5a669a48813
diff --git a/packages/shared/src/server/instrumentation/index.ts b/packages/shared/src/server/instrumentation/index.tsindex 2106f8ac3a..acf0640d9d 100644--- a/packages/shared/src/server/instrumentation/index.ts+++ b/packages/shared/src/server/instrumentation/index.ts@@ -319,9 +319,9 @@ export const flushMetricsToCloudWatch = () => {// CloudWatch metric name (excluding CW_UNFLATTENED_TAGS). Other metrics are// unaffected.const CW_TAG_FLATTENED_SUFFIXES = [".depth", ".rate", ".dlq_oldest_age"];-// "reason" is a Datadog-only breakdown; flattening it would rename existing-// CloudWatch series.-const CW_UNFLATTENED_TAGS = new Set(["unit", "reason"]);+// "reason" and "attempt" are Datadog-only breakdowns; flattening them would+// rename existing CloudWatch series.+const CW_UNFLATTENED_TAGS = new Set(["unit", "reason", "attempt"]);function buildCloudWatchKey(stat: string,diff --git a/worker/src/__tests__/workerManager.test.ts b/worker/src/__tests__/workerManager.test.tsindex 3526ceda3c..540f2c668d 100644--- a/worker/src/__tests__/workerManager.test.ts+++ b/worker/src/__tests__/workerManager.test.ts@@ -24,6 +24,12 @@ const computeDlqOldestAgeMs = (jobs: unknown[], nowMs: number): number =>nowMs,);+const computeWaitTimeMs = (job: unknown, nowMs: number) =>+ WorkerManager.computeWaitTimeMs(+ job as Parameters<typeof WorkerManager.computeWaitTimeMs>[0],+ nowMs,+ );+const resolveMetricInfo = (queueName: QueueName) =>(WorkerManager as unknown as {@@ -111,6 +117,96 @@ describe("WorkerManager", () => {});});+ describe("computeWaitTimeMs", () => {+ // BullMQ increments attemptsStarted when it moves the job to active, so+ // the processor sees 1 on the first pickup.+ const firstPickup = {+ timestamp: 1_000,+ opts: {},+ attemptsMade: 0,+ attemptsStarted: 1,+ stalledCounter: 0,+ };++ it("measures a first attempt from job creation", () => {+ expect(computeWaitTimeMs(firstPickup, 4_000)).toEqual({+ attempt: "first",+ waitMs: 3_000,+ });+ });++ it("excludes an intentional delay from a delayed job's wait", () => {+ // BullMQ resets job.delay to 0 when it promotes the job, so only+ // opts.delay still carries the original delay at pickup.+ expect(+ computeWaitTimeMs(+ { ...firstPickup, delay: 0, opts: { delay: 10_000 } },+ 12_500,+ ),+ ).toEqual({ attempt: "first", waitMs: 1_500 });+ });++ it("clamps to 0 when a delayed job is picked up early", () => {+ expect(+ computeWaitTimeMs({ ...firstPickup, opts: { delay: 10_000 } }, 10_000),+ ).toEqual({ attempt: "first", waitMs: 0 });+ });++ it("does not fold earlier attempts and backoff into a retry's wait", () => {+ // Created at 1s, ran for an hour, failed, backed off, picked up again.+ expect(+ computeWaitTimeMs(+ {+ ...firstPickup,+ attemptsMade: 1,+ attemptsStarted: 2,+ processedOn: 3_700_000,+ },+ 3_700_000,+ ),+ ).toEqual({ attempt: "retry", waitMs: undefined });+ });++ it("treats a stall re-run as a retry even though attemptsMade is 0", () => {+ // The worker host died mid-run; the stalled checker moved the job back+ // to wait without counting a failed attempt.+ expect(+ computeWaitTimeMs(+ {+ ...firstPickup,+ attemptsMade: 0,+ attemptsStarted: 2,+ stalledCounter: 1,+ },+ 7_200_000,+ ),+ ).toEqual({ attempt: "retry", waitMs: undefined });+ });++ it("treats a re-pickup without a stall count as a retry", () => {+ // e.g. moveJobFromActiveToWait or a manual job.retry() that keeps+ // attemptsStarted.+ expect(+ computeWaitTimeMs({ ...firstPickup, attemptsStarted: 3 }, 7_200_000),+ ).toEqual({ attempt: "retry", waitMs: undefined });+ });++ it.each([+ ["attemptsMade", { attemptsMade: 1, stalledCounter: 0 }],+ ["stalledCounter", { attemptsMade: 0, stalledCounter: 1 }],+ ])(+ "detects a retry from %s alone when attemptsStarted is missing",+ (_field, counters) => {+ expect(+ computeWaitTimeMs(+ { timestamp: 1_000, opts: {}, ...counters },+ 7_200_000,+ ),+ ).toEqual({ attempt: "retry", waitMs: undefined });+ },+ );+ });+describe("dlq oldest job lookup", () => {it("getFailed returns newest-first, so index -1 is the oldest job", async () => {const queueName = `dlq-oldest-age-${randomUUID()}`;diff --git a/worker/src/__tests__/workerManagerMetrics.test.ts b/worker/src/__tests__/workerManagerMetrics.test.tsindex d03eb47976..fc7ad906f0 100644--- a/worker/src/__tests__/workerManagerMetrics.test.ts+++ b/worker/src/__tests__/workerManagerMetrics.test.ts@@ -3,7 +3,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest";const mocks = vi.hoisted(() => ({handlers: new Map<string, (...args: unknown[]) => void>(),processor: undefined as- | ((job: { data: unknown; timestamp: number }) => Promise<unknown>)+ | ((job: {+ data: unknown;+ timestamp: number;+ attemptsMade?: number;+ attemptsStarted?: number;+ }) => Promise<unknown>)| undefined,legacyRecordGauge: vi.fn(),legacyRecordHistogram: vi.fn(),@@ -102,7 +107,11 @@ describe("WorkerManager queue metrics", () => {await Promise.resolve();expect(mocks.recordIncrement.mock.calls).toEqual([- ["langfuse.queue.trace_delete.rate", 1, { type: "request" }],+ [+ "langfuse.queue.trace_delete.rate",+ 1,+ { type: "request", attempt: "first" },+ ],]);expect(mocks.recordDistribution.mock.calls.map(([metric, _value, tags]) => [@@ -112,7 +121,7 @@ describe("WorkerManager queue metrics", () => {).toEqual([["langfuse.queue.trace_delete.time_distribution",- { type: "wait", unit: "milliseconds" },+ { type: "wait", attempt: "first", unit: "milliseconds" },],["langfuse.queue.trace_delete.time_distribution",@@ -141,6 +150,29 @@ describe("WorkerManager queue metrics", () => {]);});+ it("tags a retry's request and records no wait sample for it", async () => {+ // Own queue name: register() no-ops on the statically registered one above.+ WorkerManager.register("dataset-delete" as never, async () => "processed");++ await mocks.processor!({+ data: { payload: { projectId: "project-id" } },+ timestamp: Date.now() - 3_600_000,+ attemptsMade: 1,+ attemptsStarted: 2,+ });++ expect(mocks.recordIncrement.mock.calls).toEqual([+ [+ "langfuse.queue.dataset_delete.rate",+ 1,+ { type: "request", attempt: "retry" },+ ],+ ]);+ expect(+ mocks.recordDistribution.mock.calls.map(([, , tags]) => tags.type),+ ).toEqual(["processing"]);+ });+it("emits a rate counter for completed jobs but not for active ones", () => {// Own queue name: register() no-ops on the statically registered one above.WorkerManager.register("trace-upsert" as never, async () => "processed");diff --git a/worker/src/queues/workerManager.ts b/worker/src/queues/workerManager.tsindex c986b71f51..ebcaabf4d9 100644--- a/worker/src/queues/workerManager.ts+++ b/worker/src/queues/workerManager.ts@@ -70,6 +70,33 @@ export class WorkerManager {return oldest ? nowMs - (oldest.finishedOn ?? oldest.timestamp) : 0;}+ /**+ * Queue wait before the current pickup: first attempts measure from+ * timestamp + opts.delay; retries and stall re-runs return undefined because+ * BullMQ keeps no record of when they became ready again.+ */+ public static computeWaitTimeMs(+ job: Pick<+ Job,+ | "timestamp"+ | "opts"+ | "attemptsMade"+ | "attemptsStarted"+ | "stalledCounter"+ >,+ nowMs: number,+ ): { attempt: "first" | "retry"; waitMs: number | undefined } {+ const isRetry =+ (job.attemptsMade ?? 0) > 0 ||+ (job.attemptsStarted ?? 0) > 1 ||+ (job.stalledCounter ?? 0) > 0;+ if (isRetry) {+ return { attempt: "retry", waitMs: undefined };+ }+ const readyAt = job.timestamp + Math.max(0, job.opts?.delay ?? 0);+ return { attempt: "first", waitMs: Math.max(0, nowMs - readyAt) };+ }+private static metricWrapper(processor: Processor,queueName: QueueName,@@ -78,18 +105,25 @@ export class WorkerManager {return async (job: Job) => {const startTime = Date.now();- const waitTime = Date.now() - job.timestamp;+ const { attempt, waitMs } = WorkerManager.computeWaitTimeMs(+ job,+ startTime,+ );recordIncrement(baseMetric + ".rate", 1, {type: "request",+ attempt,...shardTag,});- recordDistribution(baseMetric + ".time_distribution", waitTime, {- type: "wait",- unit: "milliseconds",- ...shardTag,- });+ if (waitMs !== undefined) {+ recordDistribution(baseMetric + ".time_distribution", waitMs, {+ type: "wait",+ attempt,+ unit: "milliseconds",+ ...shardTag,+ });+ }const clickHouseCtx = contextWithLangfuseProps({projectId: WorkerManager.extractProjectId(job),diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.tsindex 846350b7e1..606d0360f6 100644--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts@@ -512,6 +512,38 @@ describe("ClickhouseWriter", () => {expect(writer["queue"][TableName.Traces]).toHaveLength(0);});+ it("measures a requeued record's wait from the requeue, not from creation", async () => {+ const distributionSpy = vi.spyOn(serverExports, "recordDistribution");+ const slowFailureMs = 60_000;+ vi.spyOn(clickhouseClientMock, "insert")+ .mockImplementationOnce(+ () =>+ new Promise((_, reject) =>+ setTimeout(() => reject(new Error("DB Error")), slowFailureMs),+ ),+ )+ .mockResolvedValueOnce();++ writer.addToQueue(TableName.Traces, { id: "1", name: "test" });++ await vi.advanceTimersByTimeAsync(writer.writeInterval + slowFailureMs);+ expect(writer["queue"][TableName.Traces]).toHaveLength(1);+ await vi.advanceTimersByTimeAsync(writer.writeInterval);+ expect(writer["queue"][TableName.Traces]).toHaveLength(0);++ const waits = distributionSpy.mock.calls.filter(+ ([name, , tags]) =>+ name === "langfuse.queue.clickhouse_writer.time_distribution" &&+ (tags as Record<string, string>).type === "wait",+ );+ expect(waits).toHaveLength(2);+ expect(waits[0][2]).toMatchObject({ attempt: "first" });+ expect(waits[1][2]).toMatchObject({ attempt: "retry" });+ // The retry waited at most one write interval after being requeued; the+ // 60s spent in the failed insert is not wait.+ expect(waits[1][1]).toBeLessThanOrEqual(writer.writeInterval);+ });+it("should drop records after max attempts", async () => {const mockInsert = vi.spyOn(clickhouseClientMock, "insert")@@ -780,6 +812,7 @@ describe("ClickhouseWriter", () => {{entity_type: TableName.Traces,type: "wait",+ attempt: "first",format: "json",unit: "milliseconds",},diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.tsindex ba4c119d89..b58d57e857 100644--- a/worker/src/services/ClickhouseWriter/index.ts+++ b/worker/src/services/ClickhouseWriter/index.ts@@ -299,7 +299,7 @@ export class ClickhouseWriter<// Log wait timequeueItems.forEach((item) => {- const waitTime = Date.now() - item.createdAt;+ const waitTime = Date.now() - item.enqueuedAt;recordHistogram("langfuse.queue.clickhouse_writer.wait_time", waitTime, {format: this.strategyFactory.format,unit: "milliseconds",@@ -310,6 +310,7 @@ export class ClickhouseWriter<{entity_type: tableName,type: "wait",+ attempt: item.requeued ? "retry" : "first",format: this.strategyFactory.format,unit: "milliseconds",},@@ -384,7 +385,14 @@ export class ClickhouseWriter<// Prepend requeue items to the front of the queue to maintain order as much as possible with parallel execution.if (requeueItems.length > 0) {- entityQueue.unshift(...requeueItems);+ const requeuedAt = Date.now();+ entityQueue.unshift(+ ...requeueItems.map((item) => ({+ ...item,+ enqueuedAt: requeuedAt,+ requeued: true,+ })),+ );}currentSpan?.addEvent("clickhouse-query-split-retry", {@@ -482,6 +490,8 @@ export class ClickhouseWriter<entityQueue.push({...item,attempts: item.attempts + 1,+ enqueuedAt: Date.now(),+ requeued: true,});} else {// TODO - Add to a dead letter queue in Redis rather than dropping@@ -546,7 +556,8 @@ export class ClickhouseWriter<const entityQueue = this.queue[tableName];entityQueue.push({- createdAt: Date.now(),+ enqueuedAt: Date.now(),+ requeued: false,attempts: 1,data,});@@ -621,7 +632,9 @@ type ClickhouseQueue<PayloadMap extends WriterPayloadMap> = {};type ClickhouseWriterQueueItem<Payload> = {- createdAt: number;+ /** When the item last entered the in-memory queue, including requeues. */+ enqueuedAt: number;+ requeued: boolean;attempts: number;data: Payload;};