🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(in-app-agent): retry failed BullMQ jobs on the consuming workers (#16337)
Lysander Kiesel5297c470ce4c
diff --git a/worker/src/app.ts b/worker/src/app.tsindex 6b0245653d..9c908a957f 100644--- a/worker/src/app.ts+++ b/worker/src/app.ts@@ -101,6 +101,7 @@ import { QueueMetricsRunner } from "./features/queue-metrics-runner";import { MonitorRunner } from "./features/monitor-runner";import { DeletedMaskCleaner } from "./features/deleted-mask-cleaner";import { TraceDeleteBatchActionRunner } from "./features/trace-delete-batch-action-runner";+import { InAppAgentDlqRetryRunner } from "./features/in-app-agent-dlq-retry-runner";const app = express();@@ -425,6 +426,8 @@ if (env.QUEUE_CONSUMER_MONITOR_QUEUE_IS_ENABLED === "true") {});}+export let inAppAgentDlqRetryRunner: InAppAgentDlqRetryRunner | null = null;+if (env.QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED === "true") {WorkerManager.register(QueueName.InAppAgentRunQueue,@@ -438,6 +441,9 @@ if (env.QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED === "true") {maxStalledCount: 0,},);++ inAppAgentDlqRetryRunner = new InAppAgentDlqRetryRunner();+ inAppAgentDlqRetryRunner.start();}// Cloud Spend Alert Queue: Only enable in cloud environment with Stripediff --git a/worker/src/env.ts b/worker/src/env.tsindex ecc061930d..a3ac45e9d6 100644--- a/worker/src/env.ts+++ b/worker/src/env.ts@@ -651,6 +651,10 @@ const EnvSchema = z.object({.number().positive().default(5),+ LANGFUSE_IN_APP_AGENT_DLQ_RETRY_INTERVAL_MS: z.coerce+ .number()+ .positive()+ .default(600_000),LANGFUSE_DELETE_BATCH_SIZE: z.coerce.number().positive().default(2000),LANGFUSE_TOKEN_COUNT_WORKER_POOL_SIZE: z.coerce.number()diff --git a/worker/src/features/in-app-agent-dlq-retry-runner/index.test.ts b/worker/src/features/in-app-agent-dlq-retry-runner/index.test.tsnew file mode 100644index 0000000000..b6258d7ef7--- /dev/null+++ b/worker/src/features/in-app-agent-dlq-retry-runner/index.test.ts@@ -0,0 +1,108 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ getQueue: vi.fn(),+}));++vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),+ getQueue: mocks.getQueue,+ logger: { debug: vi.fn(), error: vi.fn(), info: vi.fn(), warn: vi.fn() },+}));++import { InAppAgentDlqRetryRunner } from "./index";++const HEARTBEAT_STALE_MS = 60_000;++function runnerWithStubbedLock() {+ const runner = new InAppAgentDlqRetryRunner();+ (runner as unknown as { lock: { withLock: unknown } }).lock = {+ withLock: async (operation: () => Promise<unknown>) => operation(),+ };+ return runner;+}++function failedJob(params: {+ id: string;+ finishedOn: number | undefined;+ retry: () => Promise<void>;+}) {+ return {+ id: params.id,+ finishedOn: params.finishedOn,+ retry: params.retry,+ };+}++describe("InAppAgentDlqRetryRunner", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("retries a failed job older than the heartbeat-stale window", async () => {+ const retry = vi.fn().mockResolvedValue(undefined);+ mocks.getQueue.mockReturnValue({+ getFailed: vi.fn().mockResolvedValue([+ failedJob({+ id: "old-job",+ finishedOn: Date.now() - HEARTBEAT_STALE_MS - 1,+ retry,+ }),+ ]),+ });++ await runnerWithStubbedLock().processBatch();++ expect(retry).toHaveBeenCalledOnce();+ });++ it("skips a job that failed within the heartbeat-stale window", async () => {+ const retry = vi.fn().mockResolvedValue(undefined);+ mocks.getQueue.mockReturnValue({+ getFailed: vi.fn().mockResolvedValue([+ failedJob({+ id: "fresh-job",+ finishedOn: Date.now() - 5_000,+ retry,+ }),+ ]),+ });++ await runnerWithStubbedLock().processBatch();++ expect(retry).not.toHaveBeenCalled();+ });++ it("continues after a retry throw so later jobs still run", async () => {+ const failingRetry = vi.fn().mockRejectedValue(new Error("retry failed"));+ const succeedingRetry = vi.fn().mockResolvedValue(undefined);+ const oldFinishedOn = Date.now() - HEARTBEAT_STALE_MS - 1;+ mocks.getQueue.mockReturnValue({+ getFailed: vi.fn().mockResolvedValue([+ failedJob({+ id: "failing-job",+ finishedOn: oldFinishedOn,+ retry: failingRetry,+ }),+ failedJob({+ id: "ok-job",+ finishedOn: oldFinishedOn,+ retry: succeedingRetry,+ }),+ ]),+ });++ await runnerWithStubbedLock().processBatch();++ expect(failingRetry).toHaveBeenCalledOnce();+ expect(succeedingRetry).toHaveBeenCalledOnce();+ });++ it("is a no-op when the queue instance is missing", async () => {+ mocks.getQueue.mockReturnValue(null);++ await expect(+ runnerWithStubbedLock().processBatch(),+ ).resolves.toBeUndefined();+ });+});diff --git a/worker/src/features/in-app-agent-dlq-retry-runner/index.ts b/worker/src/features/in-app-agent-dlq-retry-runner/index.tsnew file mode 100644index 0000000000..e0337c49af--- /dev/null+++ b/worker/src/features/in-app-agent-dlq-retry-runner/index.ts@@ -0,0 +1,92 @@+import type { Job } from "bullmq";+import { IN_APP_AGENT_HEARTBEAT_STALE_MS } from "@langfuse/shared/in-app-agent/server/tunables";+import { getQueue, logger, QueueName } from "@langfuse/shared/src/server";++import { env } from "../../env";+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";++export const IN_APP_AGENT_DLQ_RETRY_LOCK_KEY =+ "langfuse:in-app-agent-dlq-retry";++const LOCK_TTL_SECONDS = 120;++/**+ * Pokes failed in-app-agent BullMQ jobs so stall corpses can claim-miss,+ * reconcile, and ACK. BullMQ is delivery-only on this queue (`attempts: 1`);+ * this runner is the explicit retry, not the shared dead-letter cron.+ *+ * Skips jobs that failed within the heartbeat-stale window: claim-miss ACKs+ * even when reconcile writes nothing, so a too-fresh persist failure would+ * leave Postgres RUNNING with no Redis job left to poke.+ */+export class InAppAgentDlqRetryRunner extends PeriodicExclusiveRunner {+ protected get defaultIntervalMs(): number {+ return env.LANGFUSE_IN_APP_AGENT_DLQ_RETRY_INTERVAL_MS;+ }++ constructor() {+ super({+ name: "InAppAgentDlqRetryRunner",+ metricName: "in_app_agent_dlq_retry",+ lockKey: IN_APP_AGENT_DLQ_RETRY_LOCK_KEY,+ lockTtlSeconds: LOCK_TTL_SECONDS,+ onUnavailable: "fail",+ });+ }++ public override start(): void {+ logger.info(`Starting ${this.instanceName}`, {+ intervalMs: env.LANGFUSE_IN_APP_AGENT_DLQ_RETRY_INTERVAL_MS,+ });+ super.start();+ }++ protected async execute(): Promise<void> {+ await this.withLock(async () => {+ const queue = getQueue(QueueName.InAppAgentRunQueue);+ if (!queue) {+ logger.error(`${this.instanceName}: Queue not found`);+ return;+ }++ const failedJobs = await queue.getFailed();+ const nowMs = Date.now();+ let retried = 0;+ let skipped = 0;++ for (const job of failedJobs) {+ if (!isFailedInAppAgentJobReadyToRetry(job, nowMs)) {+ skipped += 1;+ continue;+ }++ try {+ await job.retry();+ retried += 1;+ } catch (error) {+ logger.error(+ `${this.instanceName}: Failed to retry job ${job.id} in ${QueueName.InAppAgentRunQueue}`,+ error,+ );+ }+ }++ logger.info(`${this.instanceName}: processed failed jobs`, {+ failed: failedJobs.length,+ retried,+ skipped,+ });+ });+ }+}++export function isFailedInAppAgentJobReadyToRetry(+ job: Pick<Job, "finishedOn">,+ nowMs: number,+): boolean {+ if (job.finishedOn == null) {+ return true;+ }++ return nowMs - job.finishedOn >= IN_APP_AGENT_HEARTBEAT_STALE_MS;+}diff --git a/worker/src/services/dlq/dlqRetryService.ts b/worker/src/services/dlq/dlqRetryService.tsindex ca7be526fb..04706b2765 100644--- a/worker/src/services/dlq/dlqRetryService.ts+++ b/worker/src/services/dlq/dlqRetryService.ts@@ -12,7 +12,6 @@ export class DlqRetryService {QueueName.ScoreDelete,QueueName.BatchActionQueue,QueueName.DataRetentionProcessingQueue,- QueueName.InAppAgentRunQueue,] as const;// called each 10 minutes, defined by the bull cron jobdiff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.tsindex acc5ea6589..69c8e1672c 100644--- a/worker/src/utils/shutdown.ts+++ b/worker/src/utils/shutdown.ts@@ -22,6 +22,7 @@ import {deletedMaskCleaner,queueMetricsRunner,monitorRunners,+ inAppAgentDlqRetryRunner,} from "../app";export const onShutdown: NodeJS.SignalsListener = async (signal) => {@@ -68,6 +69,8 @@ export const onShutdown: NodeJS.SignalsListener = async (signal) => {runner.stop();}+ inAppAgentDlqRetryRunner?.stop();+// Before closeWorkers(), while the registry is still populated (LFE-10388).logInFlightBlobExportsOnShutdown();