Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(in-app-agent): retry failed BullMQ jobs on the consuming workers (#16337)

Lysander Kiesel5297c470ce4c
diff --git a/worker/src/app.ts b/worker/src/app.ts
index 6b0245653d..9c908a957f 100644
--- a/worker/src/app.ts
+++ b/worker/src/app.ts
@@ -101,6 +101,7 @@ import { QueueMetricsRunner } from "./features/queue-metrics-runner";
import { MonitorRunner } from "./features/monitor-runner";
import { DeletedMaskCleaner } from "./features/deleted-mask-cleaner";
import { TraceDeleteBatchActionRunner } from "./features/trace-delete-batch-action-runner";
+import { InAppAgentDlqRetryRunner } from "./features/in-app-agent-dlq-retry-runner";
const app = express();
@@ -425,6 +426,8 @@ if (env.QUEUE_CONSUMER_MONITOR_QUEUE_IS_ENABLED === "true") {
});
}
+export let inAppAgentDlqRetryRunner: InAppAgentDlqRetryRunner | null = null;
+
if (env.QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED === "true") {
WorkerManager.register(
QueueName.InAppAgentRunQueue,
@@ -438,6 +441,9 @@ if (env.QUEUE_CONSUMER_IN_APP_AGENT_RUN_QUEUE_IS_ENABLED === "true") {
maxStalledCount: 0,
},
);
+
+ inAppAgentDlqRetryRunner = new InAppAgentDlqRetryRunner();
+ inAppAgentDlqRetryRunner.start();
}
// Cloud Spend Alert Queue: Only enable in cloud environment with Stripe
diff --git a/worker/src/env.ts b/worker/src/env.ts
index ecc061930d..a3ac45e9d6 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -651,6 +651,10 @@ const EnvSchema = z.object({
.number()
.positive()
.default(5),
+ LANGFUSE_IN_APP_AGENT_DLQ_RETRY_INTERVAL_MS: z.coerce
+ .number()
+ .positive()
+ .default(600_000),
LANGFUSE_DELETE_BATCH_SIZE: z.coerce.number().positive().default(2000),
LANGFUSE_TOKEN_COUNT_WORKER_POOL_SIZE: z.coerce
.number()
diff --git a/worker/src/features/in-app-agent-dlq-retry-runner/index.test.ts b/worker/src/features/in-app-agent-dlq-retry-runner/index.test.ts
new file mode 100644
index 0000000000..b6258d7ef7
--- /dev/null
+++ b/worker/src/features/in-app-agent-dlq-retry-runner/index.test.ts
@@ -0,0 +1,108 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ getQueue: vi.fn(),
+}));
+
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
+ getQueue: mocks.getQueue,
+ logger: { debug: vi.fn(), error: vi.fn(), info: vi.fn(), warn: vi.fn() },
+}));
+
+import { InAppAgentDlqRetryRunner } from "./index";
+
+const HEARTBEAT_STALE_MS = 60_000;
+
+function runnerWithStubbedLock() {
+ const runner = new InAppAgentDlqRetryRunner();
+ (runner as unknown as { lock: { withLock: unknown } }).lock = {
+ withLock: async (operation: () => Promise<unknown>) => operation(),
+ };
+ return runner;
+}
+
+function failedJob(params: {
+ id: string;
+ finishedOn: number | undefined;
+ retry: () => Promise<void>;
+}) {
+ return {
+ id: params.id,
+ finishedOn: params.finishedOn,
+ retry: params.retry,
+ };
+}
+
+describe("InAppAgentDlqRetryRunner", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it("retries a failed job older than the heartbeat-stale window", async () => {
+ const retry = vi.fn().mockResolvedValue(undefined);
+ mocks.getQueue.mockReturnValue({
+ getFailed: vi.fn().mockResolvedValue([
+ failedJob({
+ id: "old-job",
+ finishedOn: Date.now() - HEARTBEAT_STALE_MS - 1,
+ retry,
+ }),
+ ]),
+ });
+
+ await runnerWithStubbedLock().processBatch();
+
+ expect(retry).toHaveBeenCalledOnce();
+ });
+
+ it("skips a job that failed within the heartbeat-stale window", async () => {
+ const retry = vi.fn().mockResolvedValue(undefined);
+ mocks.getQueue.mockReturnValue({
+ getFailed: vi.fn().mockResolvedValue([
+ failedJob({
+ id: "fresh-job",
+ finishedOn: Date.now() - 5_000,
+ retry,
+ }),
+ ]),
+ });
+
+ await runnerWithStubbedLock().processBatch();
+
+ expect(retry).not.toHaveBeenCalled();
+ });
+
+ it("continues after a retry throw so later jobs still run", async () => {
+ const failingRetry = vi.fn().mockRejectedValue(new Error("retry failed"));
+ const succeedingRetry = vi.fn().mockResolvedValue(undefined);
+ const oldFinishedOn = Date.now() - HEARTBEAT_STALE_MS - 1;
+ mocks.getQueue.mockReturnValue({
+ getFailed: vi.fn().mockResolvedValue([
+ failedJob({
+ id: "failing-job",
+ finishedOn: oldFinishedOn,
+ retry: failingRetry,
+ }),
+ failedJob({
+ id: "ok-job",
+ finishedOn: oldFinishedOn,
+ retry: succeedingRetry,
+ }),
+ ]),
+ });
+
+ await runnerWithStubbedLock().processBatch();
+
+ expect(failingRetry).toHaveBeenCalledOnce();
+ expect(succeedingRetry).toHaveBeenCalledOnce();
+ });
+
+ it("is a no-op when the queue instance is missing", async () => {
+ mocks.getQueue.mockReturnValue(null);
+
+ await expect(
+ runnerWithStubbedLock().processBatch(),
+ ).resolves.toBeUndefined();
+ });
+});
diff --git a/worker/src/features/in-app-agent-dlq-retry-runner/index.ts b/worker/src/features/in-app-agent-dlq-retry-runner/index.ts
new file mode 100644
index 0000000000..e0337c49af
--- /dev/null
+++ b/worker/src/features/in-app-agent-dlq-retry-runner/index.ts
@@ -0,0 +1,92 @@
+import type { Job } from "bullmq";
+import { IN_APP_AGENT_HEARTBEAT_STALE_MS } from "@langfuse/shared/in-app-agent/server/tunables";
+import { getQueue, logger, QueueName } from "@langfuse/shared/src/server";
+
+import { env } from "../../env";
+import { PeriodicExclusiveRunner } from "../../utils/PeriodicExclusiveRunner";
+
+export const IN_APP_AGENT_DLQ_RETRY_LOCK_KEY =
+ "langfuse:in-app-agent-dlq-retry";
+
+const LOCK_TTL_SECONDS = 120;
+
+/**
+ * Pokes failed in-app-agent BullMQ jobs so stall corpses can claim-miss,
+ * reconcile, and ACK. BullMQ is delivery-only on this queue (`attempts: 1`);
+ * this runner is the explicit retry, not the shared dead-letter cron.
+ *
+ * Skips jobs that failed within the heartbeat-stale window: claim-miss ACKs
+ * even when reconcile writes nothing, so a too-fresh persist failure would
+ * leave Postgres RUNNING with no Redis job left to poke.
+ */
+export class InAppAgentDlqRetryRunner extends PeriodicExclusiveRunner {
+ protected get defaultIntervalMs(): number {
+ return env.LANGFUSE_IN_APP_AGENT_DLQ_RETRY_INTERVAL_MS;
+ }
+
+ constructor() {
+ super({
+ name: "InAppAgentDlqRetryRunner",
+ metricName: "in_app_agent_dlq_retry",
+ lockKey: IN_APP_AGENT_DLQ_RETRY_LOCK_KEY,
+ lockTtlSeconds: LOCK_TTL_SECONDS,
+ onUnavailable: "fail",
+ });
+ }
+
+ public override start(): void {
+ logger.info(`Starting ${this.instanceName}`, {
+ intervalMs: env.LANGFUSE_IN_APP_AGENT_DLQ_RETRY_INTERVAL_MS,
+ });
+ super.start();
+ }
+
+ protected async execute(): Promise<void> {
+ await this.withLock(async () => {
+ const queue = getQueue(QueueName.InAppAgentRunQueue);
+ if (!queue) {
+ logger.error(`${this.instanceName}: Queue not found`);
+ return;
+ }
+
+ const failedJobs = await queue.getFailed();
+ const nowMs = Date.now();
+ let retried = 0;
+ let skipped = 0;
+
+ for (const job of failedJobs) {
+ if (!isFailedInAppAgentJobReadyToRetry(job, nowMs)) {
+ skipped += 1;
+ continue;
+ }
+
+ try {
+ await job.retry();
+ retried += 1;
+ } catch (error) {
+ logger.error(
+ `${this.instanceName}: Failed to retry job ${job.id} in ${QueueName.InAppAgentRunQueue}`,
+ error,
+ );
+ }
+ }
+
+ logger.info(`${this.instanceName}: processed failed jobs`, {
+ failed: failedJobs.length,
+ retried,
+ skipped,
+ });
+ });
+ }
+}
+
+export function isFailedInAppAgentJobReadyToRetry(
+ job: Pick<Job, "finishedOn">,
+ nowMs: number,
+): boolean {
+ if (job.finishedOn == null) {
+ return true;
+ }
+
+ return nowMs - job.finishedOn >= IN_APP_AGENT_HEARTBEAT_STALE_MS;
+}
diff --git a/worker/src/services/dlq/dlqRetryService.ts b/worker/src/services/dlq/dlqRetryService.ts
index ca7be526fb..04706b2765 100644
--- a/worker/src/services/dlq/dlqRetryService.ts
+++ b/worker/src/services/dlq/dlqRetryService.ts
@@ -12,7 +12,6 @@ export class DlqRetryService {
QueueName.ScoreDelete,
QueueName.BatchActionQueue,
QueueName.DataRetentionProcessingQueue,
- QueueName.InAppAgentRunQueue,
] as const;
// called each 10 minutes, defined by the bull cron job
diff --git a/worker/src/utils/shutdown.ts b/worker/src/utils/shutdown.ts
index acc5ea6589..69c8e1672c 100644
--- a/worker/src/utils/shutdown.ts
+++ b/worker/src/utils/shutdown.ts
@@ -22,6 +22,7 @@ import {
deletedMaskCleaner,
queueMetricsRunner,
monitorRunners,
+ inAppAgentDlqRetryRunner,
} from "../app";
export const onShutdown: NodeJS.SignalsListener = async (signal) => {
@@ -68,6 +69,8 @@ export const onShutdown: NodeJS.SignalsListener = async (signal) => {
runner.stop();
}
+ inAppAgentDlqRetryRunner?.stop();
+
// Before closeWorkers(), while the registry is still populated (LFE-10388).
logInFlightBlobExportsOnShutdown();