🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(worker): poll queue metrics once per interval across workers (#18672)
Steffen Reuther2ff268bf3ea1
diff --git a/.agents/skills/datadog-query-recipes/references/queue-consumers.md b/.agents/skills/datadog-query-recipes/references/queue-consumers.mdindex 1e90b22fc4..aff63bf229 100644--- a/.agents/skills/datadog-query-recipes/references/queue-consumers.md+++ b/.agents/skills/datadog-query-recipes/references/queue-consumers.md@@ -207,7 +207,10 @@ Metric emission and shard handling differ by source:- `QueueMetricsRunner` emits `.depth` and `.dlq_oldest_age` only for queueinstances with a worker registered in the current process. It does not add a`shard` tag for a non-sharded queue. For a sharded queue, it emits one series- per registered physical shard plus `shard:all`.+ per registered physical shard plus `shard:all`. Each interval, only one of+ the workers that consume the same queues polls and emits, so a series'+ `task_arn` changes between points. Aggregate with `max`, never `sum` across+ tasks.For `.depth`, `shard:all` is the sum across registered shards when every pollsucceeds. If only some shard polls succeed, the runner estimates the total bydiff --git a/worker/src/env.ts b/worker/src/env.tsindex a1e6dfe7ac..ae53a57ae3 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -808,6 +808,7 @@ const EnvSchema = z.object({.number().positive().default(2),+ /** Interval at which one worker per queue set polls queue depth and DLQ age */LANGFUSE_QUEUE_METRICS_INTERVAL_MS: z.coerce.number().min(100).default(1000),LANGFUSE_QUEUE_METRICS_ENABLED: z.enum(["true", "false"]).default("true"),// Upper bound for a random delay before the worker registers its queues anddiff --git a/worker/src/features/queue-metrics-runner/index.test.ts b/worker/src/features/queue-metrics-runner/index.test.tsindex 629714c5dc..7064bb6eb6 100644--- a/worker/src/features/queue-metrics-runner/index.test.ts+++ b/worker/src/features/queue-metrics-runner/index.test.ts@@ -1,9 +1,12 @@-import { beforeEach, describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";const mocks = vi.hoisted(() => ({+ flushMetricsToCloudWatch: vi.fn(),getFailed: vi.fn(),getJobCounts: vi.fn(),recordGauge: vi.fn(),+ recordIncrement: vi.fn(),+ redisSet: vi.fn(),updateActiveIngestFailureProjectsMetric: vi.fn(),emitV4LegacyApiUsageFreshnessMetrics: vi.fn(),}));@@ -14,7 +17,11 @@ vi.mock("@langfuse/shared/src/server", () => ({},convertQueueNameToMetricName: (queueName: string) =>`langfuse.queue.${queueName.replace(/-/g, "_").replace(/_queue$/, "")}`,- instrumentAsync: vi.fn(),+ flushMetricsToCloudWatch: mocks.flushMetricsToCloudWatch,+ instrumentAsync: (+ _options: unknown,+ fn: (span: { setAttribute: () => void }) => Promise<unknown>,+ ) => fn({ setAttribute: vi.fn() }),logger: {debug: vi.fn(),error: vi.fn(),@@ -23,7 +30,8 @@ vi.mock("@langfuse/shared/src/server", () => ({},recordDistribution: vi.fn(),recordGauge: mocks.recordGauge,- recordIncrement: vi.fn(),+ recordIncrement: mocks.recordIncrement,+ redis: { set: mocks.redisSet },traceException: vi.fn(),updateActiveIngestFailureProjectsMetric:mocks.updateActiveIngestFailureProjectsMetric,@@ -61,6 +69,7 @@ vi.mock("../v4/v4LegacyApiUsageMetrics", () => ({}));import { QueueMetricsRunner } from ".";+import { WorkerManager } from "../../queues/workerManager";class TestQueueMetricsRunner extends QueueMetricsRunner {public executeOnce(): Promise<void> {@@ -71,6 +80,7 @@ class TestQueueMetricsRunner extends QueueMetricsRunner {describe("QueueMetricsRunner", () => {beforeEach(() => {vi.clearAllMocks();+ mocks.redisSet.mockResolvedValue("OK");mocks.getFailed.mockResolvedValue([]);mocks.getJobCounts.mockResolvedValue({active: 2,@@ -82,6 +92,10 @@ describe("QueueMetricsRunner", () => {mocks.emitV4LegacyApiUsageFreshnessMetrics.mockResolvedValue(undefined);});+ afterEach(() => {+ vi.useRealTimers();+ });+it("emits only the canonical gauges for a non-sharded queue", async () => {await new TestQueueMetricsRunner().executeOnce();@@ -123,5 +137,81 @@ describe("QueueMetricsRunner", () => {.map(([metricName]) => metricName).filter((metricName) => legacyMetricNames.has(metricName)),).toEqual([]);+ expect(mocks.flushMetricsToCloudWatch).toHaveBeenCalledTimes(1);+ });++ it("polls nothing when another worker already claimed the interval", async () => {+ mocks.redisSet.mockResolvedValue(null);++ await new TestQueueMetricsRunner().executeOnce();++ expect(mocks.redisSet).toHaveBeenCalledWith(+ expect.stringMatching(/^langfuse:queue-metrics-runner:claim:/),+ expect.any(String),+ "PX",+ 900,+ "NX",+ );+ expect(mocks.getJobCounts).not.toHaveBeenCalled();+ expect(mocks.getFailed).not.toHaveBeenCalled();+ expect(+ mocks.updateActiveIngestFailureProjectsMetric,+ ).not.toHaveBeenCalled();+ expect(mocks.recordGauge).not.toHaveBeenCalled();+ expect(mocks.flushMetricsToCloudWatch).not.toHaveBeenCalled();+ });++ it("polls without a claim and records a failed run when the claim errors", async () => {+ mocks.redisSet.mockRejectedValue(new Error("OOM command not allowed"));+ const runner = new QueueMetricsRunner();++ runner.start();+ await vi.waitFor(() =>+ expect(mocks.recordIncrement).toHaveBeenCalledWith(+ "langfuse.periodic_runner.completed",+ 1,+ { runner: "queue_metrics_runner", outcome: "failed" },+ ),+ );+ runner.stop();++ expect(mocks.getJobCounts).toHaveBeenCalledTimes(1);+ expect(mocks.recordGauge).toHaveBeenCalledWith(+ "langfuse.queue.trace_delete.depth",+ 12,+ { type: "waiting", unit: "records" },+ );+ });++ it("flushes CloudWatch at most once per flush floor", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(new Date("2026-01-01T00:00:00Z"));+ const runner = new TestQueueMetricsRunner();++ await runner.executeOnce();+ vi.setSystemTime(new Date("2026-01-01T00:00:09Z"));+ await runner.executeOnce();+ expect(mocks.flushMetricsToCloudWatch).toHaveBeenCalledTimes(1);++ vi.setSystemTime(new Date("2026-01-01T00:00:10Z"));+ await runner.executeOnce();+ expect(mocks.flushMetricsToCloudWatch).toHaveBeenCalledTimes(2);+ });++ it("claims separately for workers that consume different queues", async () => {+ vi.mocked(WorkerManager.getRegisteredQueueNames)+ .mockReturnValueOnce(["trace-delete"])+ .mockReturnValueOnce(["score-delete", "trace-delete"])+ .mockReturnValueOnce(["trace-delete", "score-delete"]);++ for (let i = 0; i < 3; i++) {+ await new TestQueueMetricsRunner().executeOnce();+ }++ const [single, pair, samePairReordered] = mocks.redisSet.mock.calls.map(+ ([key]) => key,+ );+ expect(pair).not.toEqual(single);+ expect(samePairReordered).toEqual(pair);});});diff --git a/worker/src/features/queue-metrics-runner/index.ts b/worker/src/features/queue-metrics-runner/index.tsindex 5897df12fd..98c89c3f25 100644--- a/worker/src/features/queue-metrics-runner/index.ts+++ b/worker/src/features/queue-metrics-runner/index.ts@@ -1,9 +1,12 @@+import { createHash, randomUUID } from "crypto";import { Queue } from "bullmq";import {QueueName,convertQueueNameToMetricName,+ flushMetricsToCloudWatch,recordGauge,+ redis,updateActiveIngestFailureProjectsMetric,logger,} from "@langfuse/shared/src/server";@@ -20,6 +23,11 @@ import { emitV4LegacyApiUsageFreshnessMetrics } from "../v4/v4LegacyApiUsageMetrimport { emitTopicStagingMetrics } from "@langfuse/shared/topics/server";type DepthType = "waiting" | "failed" | "active" | "delayed";+const INTERVAL_CLAIM_KEY_PREFIX = "langfuse:queue-metrics-runner:claim";+/** Claim TTL as a fraction of the interval, below 1 so a single worker never skips its own tick */+const CLAIM_TTL_INTERVAL_FRACTION = 0.9;+/** Minimum gap between this runner's CloudWatch flushes, so a short interval does not send one PutMetricData request per tick */+const CLOUDWATCH_FLUSH_FLOOR_MS = 10_000;const TOPICS_QUEUES = new Set([QueueName.Topics,QueueName.TopicsUpdate,@@ -87,6 +95,9 @@ function emitDepth(}export class QueueMetricsRunner extends PeriodicRunner {+ private readonly instanceId = randomUUID();+ private lastCloudWatchFlushAt = 0;+constructor() {super("queue_metrics_runner");}@@ -105,6 +116,11 @@ export class QueueMetricsRunner extends PeriodicRunner {// create unnecessary Redis connections and can trigger side effects// (e.g. CloudUsageMeteringQueue.getInstance() enqueues cron jobs).const registeredNames = new Set(WorkerManager.getRegisteredQueueNames());+ if (!(await this.claimInterval(registeredNames))) {+ this.markRunSkipped();+ return;+ }+const promises: Promise<void>[] = [];if (registeredNames.has(QueueName.Topics)) {@@ -319,5 +335,51 @@ export class QueueMetricsRunner extends PeriodicRunner {}await Promise.allSettled(promises);++ this.flushToCloudWatch();+ }++ /**+ * Queue depth and DLQ age read the same from every worker, so workers that+ * consume the same queues share one poll per interval. The claim is never+ * released: it expires, and the next worker whose timer fires polls. Workers+ * that consume different queues hold separate claims. A failed claim still+ * polls, because Redis can reject writes (e.g. full under `noeviction`)+ * while serving the reads that produce these gauges.+ */+ private async claimInterval(registeredNames: Set<string>): Promise<boolean> {+ if (!redis) return true;++ const queueSetHash = createHash("sha256")+ .update([...registeredNames].sort().join(","))+ .digest("hex")+ .slice(0, 16);++ try {+ const result = await redis.set(+ `${INTERVAL_CLAIM_KEY_PREFIX}:${queueSetHash}`,+ this.instanceId,+ "PX",+ Math.floor(this.defaultIntervalMs * CLAIM_TTL_INTERVAL_FRACTION),+ "NX",+ );+ return result === "OK";+ } catch (err) {+ this.markRunFailed(err);+ logger.error(+ "Queue metrics: failed to claim the polling interval, polling without a claim",+ err,+ );+ return true;+ }+ }++ /** Publishes this tick's gauges without waiting for the 30s CloudWatch batch, since the next tick may run on another worker */+ private flushToCloudWatch(): void {+ const now = Date.now();+ if (now - this.lastCloudWatchFlushAt < CLOUDWATCH_FLUSH_FLOOR_MS) return;++ this.lastCloudWatchFlushAt = now;+ flushMetricsToCloudWatch();}}