Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): stop counting earlier attempts as queue wait time (#19039)

Niklas Semmlerf5a669a48813
diff --git a/packages/shared/src/server/instrumentation/index.ts b/packages/shared/src/server/instrumentation/index.ts
index 2106f8ac3a..acf0640d9d 100644
--- a/packages/shared/src/server/instrumentation/index.ts
+++ b/packages/shared/src/server/instrumentation/index.ts
@@ -319,9 +319,9 @@ export const flushMetricsToCloudWatch = () => {
// CloudWatch metric name (excluding CW_UNFLATTENED_TAGS). Other metrics are
// unaffected.
const CW_TAG_FLATTENED_SUFFIXES = [".depth", ".rate", ".dlq_oldest_age"];
-// "reason" is a Datadog-only breakdown; flattening it would rename existing
-// CloudWatch series.
-const CW_UNFLATTENED_TAGS = new Set(["unit", "reason"]);
+// "reason" and "attempt" are Datadog-only breakdowns; flattening them would
+// rename existing CloudWatch series.
+const CW_UNFLATTENED_TAGS = new Set(["unit", "reason", "attempt"]);
function buildCloudWatchKey(
stat: string,
diff --git a/worker/src/__tests__/workerManager.test.ts b/worker/src/__tests__/workerManager.test.ts
index 3526ceda3c..540f2c668d 100644
--- a/worker/src/__tests__/workerManager.test.ts
+++ b/worker/src/__tests__/workerManager.test.ts
@@ -24,6 +24,12 @@ const computeDlqOldestAgeMs = (jobs: unknown[], nowMs: number): number =>
nowMs,
);
+const computeWaitTimeMs = (job: unknown, nowMs: number) =>
+ WorkerManager.computeWaitTimeMs(
+ job as Parameters<typeof WorkerManager.computeWaitTimeMs>[0],
+ nowMs,
+ );
+
const resolveMetricInfo = (queueName: QueueName) =>
(
WorkerManager as unknown as {
@@ -111,6 +117,96 @@ describe("WorkerManager", () => {
});
});
+ describe("computeWaitTimeMs", () => {
+ // BullMQ increments attemptsStarted when it moves the job to active, so
+ // the processor sees 1 on the first pickup.
+ const firstPickup = {
+ timestamp: 1_000,
+ opts: {},
+ attemptsMade: 0,
+ attemptsStarted: 1,
+ stalledCounter: 0,
+ };
+
+ it("measures a first attempt from job creation", () => {
+ expect(computeWaitTimeMs(firstPickup, 4_000)).toEqual({
+ attempt: "first",
+ waitMs: 3_000,
+ });
+ });
+
+ it("excludes an intentional delay from a delayed job's wait", () => {
+ // BullMQ resets job.delay to 0 when it promotes the job, so only
+ // opts.delay still carries the original delay at pickup.
+ expect(
+ computeWaitTimeMs(
+ { ...firstPickup, delay: 0, opts: { delay: 10_000 } },
+ 12_500,
+ ),
+ ).toEqual({ attempt: "first", waitMs: 1_500 });
+ });
+
+ it("clamps to 0 when a delayed job is picked up early", () => {
+ expect(
+ computeWaitTimeMs({ ...firstPickup, opts: { delay: 10_000 } }, 10_000),
+ ).toEqual({ attempt: "first", waitMs: 0 });
+ });
+
+ it("does not fold earlier attempts and backoff into a retry's wait", () => {
+ // Created at 1s, ran for an hour, failed, backed off, picked up again.
+ expect(
+ computeWaitTimeMs(
+ {
+ ...firstPickup,
+ attemptsMade: 1,
+ attemptsStarted: 2,
+ processedOn: 3_700_000,
+ },
+ 3_700_000,
+ ),
+ ).toEqual({ attempt: "retry", waitMs: undefined });
+ });
+
+ it("treats a stall re-run as a retry even though attemptsMade is 0", () => {
+ // The worker host died mid-run; the stalled checker moved the job back
+ // to wait without counting a failed attempt.
+ expect(
+ computeWaitTimeMs(
+ {
+ ...firstPickup,
+ attemptsMade: 0,
+ attemptsStarted: 2,
+ stalledCounter: 1,
+ },
+ 7_200_000,
+ ),
+ ).toEqual({ attempt: "retry", waitMs: undefined });
+ });
+
+ it("treats a re-pickup without a stall count as a retry", () => {
+ // e.g. moveJobFromActiveToWait or a manual job.retry() that keeps
+ // attemptsStarted.
+ expect(
+ computeWaitTimeMs({ ...firstPickup, attemptsStarted: 3 }, 7_200_000),
+ ).toEqual({ attempt: "retry", waitMs: undefined });
+ });
+
+ it.each([
+ ["attemptsMade", { attemptsMade: 1, stalledCounter: 0 }],
+ ["stalledCounter", { attemptsMade: 0, stalledCounter: 1 }],
+ ])(
+ "detects a retry from %s alone when attemptsStarted is missing",
+ (_field, counters) => {
+ expect(
+ computeWaitTimeMs(
+ { timestamp: 1_000, opts: {}, ...counters },
+ 7_200_000,
+ ),
+ ).toEqual({ attempt: "retry", waitMs: undefined });
+ },
+ );
+ });
+
describe("dlq oldest job lookup", () => {
it("getFailed returns newest-first, so index -1 is the oldest job", async () => {
const queueName = `dlq-oldest-age-${randomUUID()}`;
diff --git a/worker/src/__tests__/workerManagerMetrics.test.ts b/worker/src/__tests__/workerManagerMetrics.test.ts
index d03eb47976..fc7ad906f0 100644
--- a/worker/src/__tests__/workerManagerMetrics.test.ts
+++ b/worker/src/__tests__/workerManagerMetrics.test.ts
@@ -3,7 +3,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
handlers: new Map<string, (...args: unknown[]) => void>(),
processor: undefined as
- | ((job: { data: unknown; timestamp: number }) => Promise<unknown>)
+ | ((job: {
+ data: unknown;
+ timestamp: number;
+ attemptsMade?: number;
+ attemptsStarted?: number;
+ }) => Promise<unknown>)
| undefined,
legacyRecordGauge: vi.fn(),
legacyRecordHistogram: vi.fn(),
@@ -102,7 +107,11 @@ describe("WorkerManager queue metrics", () => {
await Promise.resolve();
expect(mocks.recordIncrement.mock.calls).toEqual([
- ["langfuse.queue.trace_delete.rate", 1, { type: "request" }],
+ [
+ "langfuse.queue.trace_delete.rate",
+ 1,
+ { type: "request", attempt: "first" },
+ ],
]);
expect(
mocks.recordDistribution.mock.calls.map(([metric, _value, tags]) => [
@@ -112,7 +121,7 @@ describe("WorkerManager queue metrics", () => {
).toEqual([
[
"langfuse.queue.trace_delete.time_distribution",
- { type: "wait", unit: "milliseconds" },
+ { type: "wait", attempt: "first", unit: "milliseconds" },
],
[
"langfuse.queue.trace_delete.time_distribution",
@@ -141,6 +150,29 @@ describe("WorkerManager queue metrics", () => {
]);
});
+ it("tags a retry's request and records no wait sample for it", async () => {
+ // Own queue name: register() no-ops on the statically registered one above.
+ WorkerManager.register("dataset-delete" as never, async () => "processed");
+
+ await mocks.processor!({
+ data: { payload: { projectId: "project-id" } },
+ timestamp: Date.now() - 3_600_000,
+ attemptsMade: 1,
+ attemptsStarted: 2,
+ });
+
+ expect(mocks.recordIncrement.mock.calls).toEqual([
+ [
+ "langfuse.queue.dataset_delete.rate",
+ 1,
+ { type: "request", attempt: "retry" },
+ ],
+ ]);
+ expect(
+ mocks.recordDistribution.mock.calls.map(([, , tags]) => tags.type),
+ ).toEqual(["processing"]);
+ });
+
it("emits a rate counter for completed jobs but not for active ones", () => {
// Own queue name: register() no-ops on the statically registered one above.
WorkerManager.register("trace-upsert" as never, async () => "processed");
diff --git a/worker/src/queues/workerManager.ts b/worker/src/queues/workerManager.ts
index c986b71f51..ebcaabf4d9 100644
--- a/worker/src/queues/workerManager.ts
+++ b/worker/src/queues/workerManager.ts
@@ -70,6 +70,33 @@ export class WorkerManager {
return oldest ? nowMs - (oldest.finishedOn ?? oldest.timestamp) : 0;
}
+ /**
+ * Queue wait before the current pickup: first attempts measure from
+ * timestamp + opts.delay; retries and stall re-runs return undefined because
+ * BullMQ keeps no record of when they became ready again.
+ */
+ public static computeWaitTimeMs(
+ job: Pick<
+ Job,
+ | "timestamp"
+ | "opts"
+ | "attemptsMade"
+ | "attemptsStarted"
+ | "stalledCounter"
+ >,
+ nowMs: number,
+ ): { attempt: "first" | "retry"; waitMs: number | undefined } {
+ const isRetry =
+ (job.attemptsMade ?? 0) > 0 ||
+ (job.attemptsStarted ?? 0) > 1 ||
+ (job.stalledCounter ?? 0) > 0;
+ if (isRetry) {
+ return { attempt: "retry", waitMs: undefined };
+ }
+ const readyAt = job.timestamp + Math.max(0, job.opts?.delay ?? 0);
+ return { attempt: "first", waitMs: Math.max(0, nowMs - readyAt) };
+ }
+
private static metricWrapper(
processor: Processor,
queueName: QueueName,
@@ -78,18 +105,25 @@ export class WorkerManager {
return async (job: Job) => {
const startTime = Date.now();
- const waitTime = Date.now() - job.timestamp;
+ const { attempt, waitMs } = WorkerManager.computeWaitTimeMs(
+ job,
+ startTime,
+ );
recordIncrement(baseMetric + ".rate", 1, {
type: "request",
+ attempt,
...shardTag,
});
- recordDistribution(baseMetric + ".time_distribution", waitTime, {
- type: "wait",
- unit: "milliseconds",
- ...shardTag,
- });
+ if (waitMs !== undefined) {
+ recordDistribution(baseMetric + ".time_distribution", waitMs, {
+ type: "wait",
+ attempt,
+ unit: "milliseconds",
+ ...shardTag,
+ });
+ }
const clickHouseCtx = contextWithLangfuseProps({
projectId: WorkerManager.extractProjectId(job),
diff --git a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
index 846350b7e1..606d0360f6 100644
--- a/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
+++ b/worker/src/services/ClickhouseWriter/ClickhouseWriter.unit.test.ts
@@ -512,6 +512,38 @@ describe("ClickhouseWriter", () => {
expect(writer["queue"][TableName.Traces]).toHaveLength(0);
});
+ it("measures a requeued record's wait from the requeue, not from creation", async () => {
+ const distributionSpy = vi.spyOn(serverExports, "recordDistribution");
+ const slowFailureMs = 60_000;
+ vi.spyOn(clickhouseClientMock, "insert")
+ .mockImplementationOnce(
+ () =>
+ new Promise((_, reject) =>
+ setTimeout(() => reject(new Error("DB Error")), slowFailureMs),
+ ),
+ )
+ .mockResolvedValueOnce();
+
+ writer.addToQueue(TableName.Traces, { id: "1", name: "test" });
+
+ await vi.advanceTimersByTimeAsync(writer.writeInterval + slowFailureMs);
+ expect(writer["queue"][TableName.Traces]).toHaveLength(1);
+ await vi.advanceTimersByTimeAsync(writer.writeInterval);
+ expect(writer["queue"][TableName.Traces]).toHaveLength(0);
+
+ const waits = distributionSpy.mock.calls.filter(
+ ([name, , tags]) =>
+ name === "langfuse.queue.clickhouse_writer.time_distribution" &&
+ (tags as Record<string, string>).type === "wait",
+ );
+ expect(waits).toHaveLength(2);
+ expect(waits[0][2]).toMatchObject({ attempt: "first" });
+ expect(waits[1][2]).toMatchObject({ attempt: "retry" });
+ // The retry waited at most one write interval after being requeued; the
+ // 60s spent in the failed insert is not wait.
+ expect(waits[1][1]).toBeLessThanOrEqual(writer.writeInterval);
+ });
+
it("should drop records after max attempts", async () => {
const mockInsert = vi
.spyOn(clickhouseClientMock, "insert")
@@ -780,6 +812,7 @@ describe("ClickhouseWriter", () => {
{
entity_type: TableName.Traces,
type: "wait",
+ attempt: "first",
format: "json",
unit: "milliseconds",
},
diff --git a/worker/src/services/ClickhouseWriter/index.ts b/worker/src/services/ClickhouseWriter/index.ts
index ba4c119d89..b58d57e857 100644
--- a/worker/src/services/ClickhouseWriter/index.ts
+++ b/worker/src/services/ClickhouseWriter/index.ts
@@ -299,7 +299,7 @@ export class ClickhouseWriter<
// Log wait time
queueItems.forEach((item) => {
- const waitTime = Date.now() - item.createdAt;
+ const waitTime = Date.now() - item.enqueuedAt;
recordHistogram("langfuse.queue.clickhouse_writer.wait_time", waitTime, {
format: this.strategyFactory.format,
unit: "milliseconds",
@@ -310,6 +310,7 @@ export class ClickhouseWriter<
{
entity_type: tableName,
type: "wait",
+ attempt: item.requeued ? "retry" : "first",
format: this.strategyFactory.format,
unit: "milliseconds",
},
@@ -384,7 +385,14 @@ export class ClickhouseWriter<
// Prepend requeue items to the front of the queue to maintain order as much as possible with parallel execution.
if (requeueItems.length > 0) {
- entityQueue.unshift(...requeueItems);
+ const requeuedAt = Date.now();
+ entityQueue.unshift(
+ ...requeueItems.map((item) => ({
+ ...item,
+ enqueuedAt: requeuedAt,
+ requeued: true,
+ })),
+ );
}
currentSpan?.addEvent("clickhouse-query-split-retry", {
@@ -482,6 +490,8 @@ export class ClickhouseWriter<
entityQueue.push({
...item,
attempts: item.attempts + 1,
+ enqueuedAt: Date.now(),
+ requeued: true,
});
} else {
// TODO - Add to a dead letter queue in Redis rather than dropping
@@ -546,7 +556,8 @@ export class ClickhouseWriter<
const entityQueue = this.queue[tableName];
entityQueue.push({
- createdAt: Date.now(),
+ enqueuedAt: Date.now(),
+ requeued: false,
attempts: 1,
data,
});
@@ -621,7 +632,9 @@ type ClickhouseQueue<PayloadMap extends WriterPayloadMap> = {
};
type ClickhouseWriterQueueItem<Payload> = {
- createdAt: number;
+ /** When the item last entered the in-memory queue, including requeues. */
+ enqueuedAt: number;
+ requeued: boolean;
attempts: number;
data: Payload;
};