Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(worker): poll queue metrics once per interval across workers (#18672)

Steffen Reuther2ff268bf3ea1
diff --git a/.agents/skills/datadog-query-recipes/references/queue-consumers.md b/.agents/skills/datadog-query-recipes/references/queue-consumers.md
index 1e90b22fc4..aff63bf229 100644
--- a/.agents/skills/datadog-query-recipes/references/queue-consumers.md
+++ b/.agents/skills/datadog-query-recipes/references/queue-consumers.md
@@ -207,7 +207,10 @@ Metric emission and shard handling differ by source:
- `QueueMetricsRunner` emits `.depth` and `.dlq_oldest_age` only for queue
instances with a worker registered in the current process. It does not add a
`shard` tag for a non-sharded queue. For a sharded queue, it emits one series
- per registered physical shard plus `shard:all`.
+ per registered physical shard plus `shard:all`. Each interval, only one of
+ the workers that consume the same queues polls and emits, so a series'
+ `task_arn` changes between points. Aggregate with `max`, never `sum` across
+ tasks.
For `.depth`, `shard:all` is the sum across registered shards when every poll
succeeds. If only some shard polls succeed, the runner estimates the total by
diff --git a/worker/src/env.ts b/worker/src/env.ts
index a1e6dfe7ac..ae53a57ae3 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -808,6 +808,7 @@ const EnvSchema = z.object({
.number()
.positive()
.default(2),
+ /** Interval at which one worker per queue set polls queue depth and DLQ age */
LANGFUSE_QUEUE_METRICS_INTERVAL_MS: z.coerce.number().min(100).default(1000),
LANGFUSE_QUEUE_METRICS_ENABLED: z.enum(["true", "false"]).default("true"),
// Upper bound for a random delay before the worker registers its queues and
diff --git a/worker/src/features/queue-metrics-runner/index.test.ts b/worker/src/features/queue-metrics-runner/index.test.ts
index 629714c5dc..7064bb6eb6 100644
--- a/worker/src/features/queue-metrics-runner/index.test.ts
+++ b/worker/src/features/queue-metrics-runner/index.test.ts
@@ -1,9 +1,12 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
+ flushMetricsToCloudWatch: vi.fn(),
getFailed: vi.fn(),
getJobCounts: vi.fn(),
recordGauge: vi.fn(),
+ recordIncrement: vi.fn(),
+ redisSet: vi.fn(),
updateActiveIngestFailureProjectsMetric: vi.fn(),
emitV4LegacyApiUsageFreshnessMetrics: vi.fn(),
}));
@@ -14,7 +17,11 @@ vi.mock("@langfuse/shared/src/server", () => ({
},
convertQueueNameToMetricName: (queueName: string) =>
`langfuse.queue.${queueName.replace(/-/g, "_").replace(/_queue$/, "")}`,
- instrumentAsync: vi.fn(),
+ flushMetricsToCloudWatch: mocks.flushMetricsToCloudWatch,
+ instrumentAsync: (
+ _options: unknown,
+ fn: (span: { setAttribute: () => void }) => Promise<unknown>,
+ ) => fn({ setAttribute: vi.fn() }),
logger: {
debug: vi.fn(),
error: vi.fn(),
@@ -23,7 +30,8 @@ vi.mock("@langfuse/shared/src/server", () => ({
},
recordDistribution: vi.fn(),
recordGauge: mocks.recordGauge,
- recordIncrement: vi.fn(),
+ recordIncrement: mocks.recordIncrement,
+ redis: { set: mocks.redisSet },
traceException: vi.fn(),
updateActiveIngestFailureProjectsMetric:
mocks.updateActiveIngestFailureProjectsMetric,
@@ -61,6 +69,7 @@ vi.mock("../v4/v4LegacyApiUsageMetrics", () => ({
}));
import { QueueMetricsRunner } from ".";
+import { WorkerManager } from "../../queues/workerManager";
class TestQueueMetricsRunner extends QueueMetricsRunner {
public executeOnce(): Promise<void> {
@@ -71,6 +80,7 @@ class TestQueueMetricsRunner extends QueueMetricsRunner {
describe("QueueMetricsRunner", () => {
beforeEach(() => {
vi.clearAllMocks();
+ mocks.redisSet.mockResolvedValue("OK");
mocks.getFailed.mockResolvedValue([]);
mocks.getJobCounts.mockResolvedValue({
active: 2,
@@ -82,6 +92,10 @@ describe("QueueMetricsRunner", () => {
mocks.emitV4LegacyApiUsageFreshnessMetrics.mockResolvedValue(undefined);
});
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
it("emits only the canonical gauges for a non-sharded queue", async () => {
await new TestQueueMetricsRunner().executeOnce();
@@ -123,5 +137,81 @@ describe("QueueMetricsRunner", () => {
.map(([metricName]) => metricName)
.filter((metricName) => legacyMetricNames.has(metricName)),
).toEqual([]);
+ expect(mocks.flushMetricsToCloudWatch).toHaveBeenCalledTimes(1);
+ });
+
+ it("polls nothing when another worker already claimed the interval", async () => {
+ mocks.redisSet.mockResolvedValue(null);
+
+ await new TestQueueMetricsRunner().executeOnce();
+
+ expect(mocks.redisSet).toHaveBeenCalledWith(
+ expect.stringMatching(/^langfuse:queue-metrics-runner:claim:/),
+ expect.any(String),
+ "PX",
+ 900,
+ "NX",
+ );
+ expect(mocks.getJobCounts).not.toHaveBeenCalled();
+ expect(mocks.getFailed).not.toHaveBeenCalled();
+ expect(
+ mocks.updateActiveIngestFailureProjectsMetric,
+ ).not.toHaveBeenCalled();
+ expect(mocks.recordGauge).not.toHaveBeenCalled();
+ expect(mocks.flushMetricsToCloudWatch).not.toHaveBeenCalled();
+ });
+
+ it("polls without a claim and records a failed run when the claim errors", async () => {
+ mocks.redisSet.mockRejectedValue(new Error("OOM command not allowed"));
+ const runner = new QueueMetricsRunner();
+
+ runner.start();
+ await vi.waitFor(() =>
+ expect(mocks.recordIncrement).toHaveBeenCalledWith(
+ "langfuse.periodic_runner.completed",
+ 1,
+ { runner: "queue_metrics_runner", outcome: "failed" },
+ ),
+ );
+ runner.stop();
+
+ expect(mocks.getJobCounts).toHaveBeenCalledTimes(1);
+ expect(mocks.recordGauge).toHaveBeenCalledWith(
+ "langfuse.queue.trace_delete.depth",
+ 12,
+ { type: "waiting", unit: "records" },
+ );
+ });
+
+ it("flushes CloudWatch at most once per flush floor", async () => {
+ vi.useFakeTimers({ toFake: ["Date"] });
+ vi.setSystemTime(new Date("2026-01-01T00:00:00Z"));
+ const runner = new TestQueueMetricsRunner();
+
+ await runner.executeOnce();
+ vi.setSystemTime(new Date("2026-01-01T00:00:09Z"));
+ await runner.executeOnce();
+ expect(mocks.flushMetricsToCloudWatch).toHaveBeenCalledTimes(1);
+
+ vi.setSystemTime(new Date("2026-01-01T00:00:10Z"));
+ await runner.executeOnce();
+ expect(mocks.flushMetricsToCloudWatch).toHaveBeenCalledTimes(2);
+ });
+
+ it("claims separately for workers that consume different queues", async () => {
+ vi.mocked(WorkerManager.getRegisteredQueueNames)
+ .mockReturnValueOnce(["trace-delete"])
+ .mockReturnValueOnce(["score-delete", "trace-delete"])
+ .mockReturnValueOnce(["trace-delete", "score-delete"]);
+
+ for (let i = 0; i < 3; i++) {
+ await new TestQueueMetricsRunner().executeOnce();
+ }
+
+ const [single, pair, samePairReordered] = mocks.redisSet.mock.calls.map(
+ ([key]) => key,
+ );
+ expect(pair).not.toEqual(single);
+ expect(samePairReordered).toEqual(pair);
});
});
diff --git a/worker/src/features/queue-metrics-runner/index.ts b/worker/src/features/queue-metrics-runner/index.ts
index 5897df12fd..98c89c3f25 100644
--- a/worker/src/features/queue-metrics-runner/index.ts
+++ b/worker/src/features/queue-metrics-runner/index.ts
@@ -1,9 +1,12 @@
+import { createHash, randomUUID } from "crypto";
import { Queue } from "bullmq";
import {
QueueName,
convertQueueNameToMetricName,
+ flushMetricsToCloudWatch,
recordGauge,
+ redis,
updateActiveIngestFailureProjectsMetric,
logger,
} from "@langfuse/shared/src/server";
@@ -20,6 +23,11 @@ import { emitV4LegacyApiUsageFreshnessMetrics } from "../v4/v4LegacyApiUsageMetr
import { emitTopicStagingMetrics } from "@langfuse/shared/topics/server";
type DepthType = "waiting" | "failed" | "active" | "delayed";
+const INTERVAL_CLAIM_KEY_PREFIX = "langfuse:queue-metrics-runner:claim";
+/** Claim TTL as a fraction of the interval, below 1 so a single worker never skips its own tick */
+const CLAIM_TTL_INTERVAL_FRACTION = 0.9;
+/** Minimum gap between this runner's CloudWatch flushes, so a short interval does not send one PutMetricData request per tick */
+const CLOUDWATCH_FLUSH_FLOOR_MS = 10_000;
const TOPICS_QUEUES = new Set([
QueueName.Topics,
QueueName.TopicsUpdate,
@@ -87,6 +95,9 @@ function emitDepth(
}
export class QueueMetricsRunner extends PeriodicRunner {
+ private readonly instanceId = randomUUID();
+ private lastCloudWatchFlushAt = 0;
+
constructor() {
super("queue_metrics_runner");
}
@@ -105,6 +116,11 @@ export class QueueMetricsRunner extends PeriodicRunner {
// create unnecessary Redis connections and can trigger side effects
// (e.g. CloudUsageMeteringQueue.getInstance() enqueues cron jobs).
const registeredNames = new Set(WorkerManager.getRegisteredQueueNames());
+ if (!(await this.claimInterval(registeredNames))) {
+ this.markRunSkipped();
+ return;
+ }
+
const promises: Promise<void>[] = [];
if (registeredNames.has(QueueName.Topics)) {
@@ -319,5 +335,51 @@ export class QueueMetricsRunner extends PeriodicRunner {
}
await Promise.allSettled(promises);
+
+ this.flushToCloudWatch();
+ }
+
+ /**
+ * Queue depth and DLQ age read the same from every worker, so workers that
+ * consume the same queues share one poll per interval. The claim is never
+ * released: it expires, and the next worker whose timer fires polls. Workers
+ * that consume different queues hold separate claims. A failed claim still
+ * polls, because Redis can reject writes (e.g. full under `noeviction`)
+ * while serving the reads that produce these gauges.
+ */
+ private async claimInterval(registeredNames: Set<string>): Promise<boolean> {
+ if (!redis) return true;
+
+ const queueSetHash = createHash("sha256")
+ .update([...registeredNames].sort().join(","))
+ .digest("hex")
+ .slice(0, 16);
+
+ try {
+ const result = await redis.set(
+ `${INTERVAL_CLAIM_KEY_PREFIX}:${queueSetHash}`,
+ this.instanceId,
+ "PX",
+ Math.floor(this.defaultIntervalMs * CLAIM_TTL_INTERVAL_FRACTION),
+ "NX",
+ );
+ return result === "OK";
+ } catch (err) {
+ this.markRunFailed(err);
+ logger.error(
+ "Queue metrics: failed to claim the polling interval, polling without a claim",
+ err,
+ );
+ return true;
+ }
+ }
+
+ /** Publishes this tick's gauges without waiting for the 30s CloudWatch batch, since the next tick may run on another worker */
+ private flushToCloudWatch(): void {
+ const now = Date.now();
+ if (now - this.lastCloudWatchFlushAt < CLOUDWATCH_FLUSH_FLOOR_MS) return;
+
+ this.lastCloudWatchFlushAt = now;
+ flushMetricsToCloudWatch();
}
}