The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
perf(chat): stop polling failed run retries (#15871)
Dottae168512f7bdb
diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.mdindex e40716791f..8177461f43 100644--- a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md+++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md@@ -444,8 +444,13 @@ Longer polling intervals alone do not complete an item.running sessions and pauses without rescheduling until an endpoint change.Tests exercise real producers with deferred fast paths disabled, retry andstartup recovery, stale-send quarantine, and zero scans after settlement.- Remaining **1s** maintenance: provider runtimes, Telegram maintenance, failed- chat-run retries, run milestone/wake-notice projection, GitHub operation+ Failed chat-run retries now also use commit wakes and durable deadlines from+ existing action rows. Issued retries honor `retryAt`; processing claims wait+ for both stale-claim expiry and any saved retry time. Startup recovers unfinished+ requests. Claim and settlement writes signal in their own short transactions,+ with agent wakeup outside those transactions. Empty and terminal queues disarm.+ Remaining **1s** maintenance: provider runtimes, Telegram maintenance,+ run milestone/wake-notice projection, GitHub operationrecovery, and Teams protocol maintenance. GitHub pre-ingress recoverymust still inspect provider history: a lost callback has no local queue row.This slice does **not** yet make the whole chat subsystem quiet. Independentdiff --git a/server/src/__tests__/app-private-hostname-gate.test.ts b/server/src/__tests__/app-private-hostname-gate.test.tsindex 8f08724191..1c46075d2a 100644--- a/server/src/__tests__/app-private-hostname-gate.test.ts+++ b/server/src/__tests__/app-private-hostname-gate.test.ts@@ -110,6 +110,7 @@ describe("createChatReconciliationCoordinator", () => {const service = readFileSync(new URL("../services/chat-channels.ts", import.meta.url), "utf8");const actionMaintenance = service.slice(service.indexOf("async function processPendingChatMaintenance("),service.indexOf("async function processPendingDeliveries("));+ expect(actionMaintenance).not.toContain("processFailedChatRunRetries(");expect(actionMaintenance).not.toContain("processPendingProviderEffects(");expect(actionMaintenance).not.toContain("processPendingGitHubWebhookIngress(");});diff --git a/server/src/__tests__/chat-action-work.test.ts b/server/src/__tests__/chat-action-work.test.tsindex b96541e79e..294715f922 100644--- a/server/src/__tests__/chat-action-work.test.ts+++ b/server/src/__tests__/chat-action-work.test.ts@@ -19,6 +19,7 @@ function setup() {const coordinator = createDeliveryWorkCoordinator({ owner, canRun: enabled, canReconcile: enabled, onError });coordinators.push(coordinator);const service = {+ processFailedChatRunRetries: vi.fn(async () => {}),processPendingProviderEffects: vi.fn(async () => {}),processPendingGitHubWebhookIngress: vi.fn(async () => {}),processPendingSlackSessionSyncs: vi.fn(async () => {}),@@ -30,7 +31,7 @@ function setup() {nextChatActionAt: vi.fn(async (): Promise<number | null> => null),nextVerificationMessageAt: vi.fn(async (): Promise<number | null> => null),};- const runs = [service.processPendingProviderEffects, service.processPendingGitHubWebhookIngress, service.processPendingSlackSessionSyncs, service.processPendingSlackBoardMessages, service.processPendingSlackTaskStarts,+ const runs = [service.processFailedChatRunRetries, service.processPendingProviderEffects, service.processPendingGitHubWebhookIngress, service.processPendingSlackSessionSyncs, service.processPendingSlackBoardMessages, service.processPendingSlackTaskStarts,service.processPendingReceiptReactions, service.processPendingSlackSessionStops, service.processPendingVerificationMessages];return { owner, service, runs, coordinator, enabled, onError, start: () => registerChatActionWork(coordinator, service, enabled) };}@@ -40,7 +41,7 @@ describe("chat action scheduling", () => {await s.start().ready;await vi.advanceTimersByTimeAsync(3_600_000);for (const run of s.runs) expect(run).toHaveBeenCalledTimes(1);- expect(s.service.nextChatActionAt).toHaveBeenCalledTimes(7);+ expect(s.service.nextChatActionAt).toHaveBeenCalledTimes(8);expect(s.service.nextVerificationMessageAt).toHaveBeenCalledTimes(1);expect(s.coordinator.nextWakeAt()).toBeNull();expect(vi.getTimerCount()).toBe(0);diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.tsindex a550acedf8..d8c8637446 100644--- a/server/src/__tests__/chat-channels.integration.test.ts+++ b/server/src/__tests__/chat-channels.integration.test.ts@@ -1,3 +1,4 @@+import { idleWorkSnapshot } from "../services/task-admission.js";import * as activityLogModule from "../services/activity-log.js";import { chatSlackManagerGrants, chatSlackRegistrations, toolOauthStates } from "@paperclipai/db";import type { PaperclipCloudConnector, SealedConnectorCredentials } from "../services/paperclip-cloud-connector.js";@@ -33026,6 +33027,26 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {await set("delivery_unknown");expect(await f.service.nextChatActionAt(kind)).toBeNull();expect(await due()).toBe(false);+ } else if (kind === "failed_run_retry") {+ expect(await due()).toBe(false);+ await set("issued");+ expect(await due()).toBe(true);+ await set("issued", { retryAt: later.toISOString() });+ expect(await f.service.nextChatActionAt(kind)).toBe(later.getTime());+ expect(await due()).toBe(false);+ await set("processing");+ expect(await f.service.nextChatActionAt(kind)).toBe(at.getTime() + 60_000);+ expect(await due()).toBe(false);+ // Both the claim lease and retry time must expire before replay.+ await set("processing", { retryAt: later.toISOString() });+ expect(await f.service.nextChatActionAt(kind)).toBe(later.getTime());+ await set("processing", { retryAt: at.toISOString() });+ expect(await f.service.nextChatActionAt(kind)).toBe(at.getTime() + 60_000);+ for (const status of ["failed", "cancelled", "processed"]) {+ await set(status, { retryable: true, retryAt: at.toISOString() });+ expect(await f.service.nextChatActionAt(kind)).toBeNull();+ expect(await due()).toBe(false);+ }} else if (kind === "slack_session_sync") {expect(await due()).toBe(true);await set("received", { retryAt: later.toISOString() });@@ -59779,6 +59800,81 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {},);+ it.each(["commit", "retry", "startup", "stale_claim", "rollback"] as const)(+ "drains failed chat retries from durable work and becomes idle: %s",+ async (mode) => {+ const context = await failedChatRetryFixture("telegram");+ const onError = vi.fn();+ const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });+ const input = { companyId: context.fixture.companyId, issueId: context.issue.id,+ agentId: context.fixture.assignedAgentId, failedRunId: context.runId, initiatedByUserId: "owner-user" };+ const stage = () => db.transaction(tx => context.service.prepareFailedChatRunRetry(tx, input));+ context.wakeup.mockClear();+ try {+ let actionId: string;+ if (mode === "startup" || mode === "stale_claim") {+ ({ actionId } = await stage());+ if (mode === "stale_claim") await db.update(chatActions).set({ status: "processing",+ updatedAt: new Date(Date.now() - 120_000) }).where(eq(chatActions.id, actionId));+ await registerChatActionWork(coordinator, context.service, () => true).ready;+ } else {+ await registerChatActionWork(coordinator, context.service, () => true).ready;+ if (mode === "rollback") {+ await expect(db.transaction(async tx => {+ await context.service.prepareFailedChatRunRetry(tx, input);+ throw new Error("rollback retry intent");+ })).rejects.toThrow("rollback retry intent");+ await vi.waitFor(() => {+ expect(coordinator.nextWakeAt()).toBeNull();+ expect(idleWorkSnapshot().active).toBe(0);+ }, { timeout: 10_000 });+ expect(context.wakeup).not.toHaveBeenCalled();+ }+ if (mode === "retry") {+ let failedAt = 0;+ context.wakeup.mockImplementationOnce(async () => {+ failedAt = Date.now();+ throw new Error("temporary admission outage");+ }).mockImplementationOnce(async () => {+ expect(Date.now() - failedAt).toBeGreaterThanOrEqual(250);+ return { accepted: true };+ });+ }+ ({ actionId } = await db.transaction(async tx => {+ const staged = await tx.transaction(nested => context.service.prepareFailedChatRunRetry(nested, input));+ // A nested savepoint must not wake the worker before the outer commit.+ await new Promise(resolve => setTimeout(resolve, 25));+ expect(context.wakeup).not.toHaveBeenCalled();+ return staged;+ }));+ }+ await vi.waitFor(async () => {+ const [action] = await db.select().from(chatActions).where(eq(chatActions.id, actionId));+ expect(action?.status).toBe("processed");+ expect(action?.result.attemptCount).toBe(mode === "retry" ? 2 : 1);+ expect(coordinator.nextWakeAt()).toBeNull();+ }, { timeout: 10_000 });+ expect(context.wakeup).toHaveBeenCalledTimes(mode === "retry" ? 2 : 1);+ const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, actionId));+ expect(receipts).toHaveLength(1);+ expect(receipts[0]?.payload).toMatchObject({ retryOfRunId: context.runId,+ wakeCommentIds: [context.action.payload.commentId] });+ // A running worker has no scheduled deadline. Wait for its final scan+ // and any commit-driven follow-up before measuring the quiet period.+ await vi.waitFor(() => {+ expect(coordinator.nextWakeAt()).toBeNull();+ expect(idleWorkSnapshot().active).toBe(0);+ }, { timeout: 10_000 });+ const scans = vi.spyOn(db, "select");+ try {+ await new Promise(resolve => setTimeout(resolve, 1200));+ expect(scans).not.toHaveBeenCalled();+ } finally { scans.mockRestore(); }+ expect(onError).not.toHaveBeenCalled();+ } finally { await coordinator.stop(); await context.service.shutdown(); }+ },+ );+it.each(["slack", "telegram"] as const)("retries the exact failed %s chat input with a new idempotent admission",async (provider) => {diff --git a/server/src/services/chat-action-work.ts b/server/src/services/chat-action-work.tsindex f794cc6a0c..6d1847526e 100644--- a/server/src/services/chat-action-work.ts+++ b/server/src/services/chat-action-work.ts@@ -8,6 +8,7 @@ import { registerChatQueueWork } from "./chat-delivery-work.js";export const SLACK_COMMAND_POST_STALE_MS = 60_000;export const SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS = 5 * 60_000;export const SLACK_COMMAND_ADMISSION_STALE_MS = 60_000;+export const FAILED_RUN_RETRY_STALE_MS = 60_000;export const PROVIDER_EFFECT_STALE_MS = 60_000;/** The dispatch predicate and scheduler use the same eligibility and deadlines.@@ -15,6 +16,9 @@ export const PROVIDER_EFFECT_STALE_MS = 60_000;export function chatActionDeadline(kind: ScheduledChatAction) {const updated = sql`extract(epoch from ${chatActions.updatedAt}) * 1000`;const retry = sql`coalesce(extract(epoch from (${chatActions.result}->>'retryAt')::timestamptz) * 1000, 0)`;+ if (kind === "failed_run_retry") return sql<number>`case+ when ${chatActions.status} = 'issued' then ${retry}+ when ${chatActions.status} = 'processing' then greatest(${updated} + ${FAILED_RUN_RETRY_STALE_MS}, ${retry}) end`;if (kind === "slack_board_message") return sql<number>`casewhen ${chatActions.status} = 'received' and ${chatEndpoints.status} not in ('paused', 'attention') then 0::numeric end`;if (kind === "slash_task_start") return sql<number>`case@@ -36,14 +40,16 @@ export function chatActionDeadline(kind: ScheduledChatAction) {function pendingChatAction(kind: ScheduledChatAction) {// Filter history before the endpoint join and aggregate. Failed actions// only remain live when the saved result permits another attempt.- const status = kind === "slack_board_message"- ? eq(chatActions.status, "received")- : kind === "slash_task_start"- ? inArray(chatActions.status, ["queued", "provider_confirmed", "admitting", "resolving", "validating"])- : kind === "slack_session_sync"- ? inArray(chatActions.status, ["received", "processing"])- : or(inArray(chatActions.status, ["received", "processing"]),- and(eq(chatActions.status, "failed"), sql`${chatActions.result}->>'retryable' = 'true'`));+ const status = kind === "failed_run_retry"+ ? inArray(chatActions.status, ["issued", "processing"])+ : kind === "slack_board_message"+ ? eq(chatActions.status, "received")+ : kind === "slash_task_start"+ ? inArray(chatActions.status, ["queued", "provider_confirmed", "admitting", "resolving", "validating"])+ : kind === "slack_session_sync"+ ? inArray(chatActions.status, ["received", "processing"])+ : or(inArray(chatActions.status, ["received", "processing"]),+ and(eq(chatActions.status, "failed"), sql`${chatActions.result}->>'retryable' = 'true'`));return and(eq(chatActions.kind, kind), status);}@@ -60,6 +66,7 @@ export async function nextChatActionAt(db: Db, kind: ScheduledChatAction): Promi}type ChatActionService = {+ processFailedChatRunRetries(): Promise<unknown>;processPendingProviderEffects(): Promise<unknown>;processPendingGitHubWebhookIngress(): Promise<unknown>;processPendingSlackSessionSyncs(): Promise<unknown>;@@ -78,6 +85,7 @@ export function registerChatActionWork(canRun: () => boolean,) {const actions = [+ { kind: "failed_run_retry", run: service.processFailedChatRunRetries },{ kind: "provider_effect", run: service.processPendingProviderEffects },{ kind: "github_webhook_ingress", run: service.processPendingGitHubWebhookIngress },{ kind: "slack_session_sync", run: service.processPendingSlackSessionSyncs },diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.tsindex 2bd74bbb9d..7716efe06a 100644--- a/server/src/services/chat-channels.ts+++ b/server/src/services/chat-channels.ts@@ -5729,7 +5729,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {lifecycleRuntimeFence,isShuttingDown: () => shuttingDown,wakeup: (...args) => options.heartbeat.wakeup(...args),- staleProcessingMs: DELIVERY_PROCESSING_STALE_MS,});const { stageTeamsBoardFileIntent, deriveTeamsFileTransferAuthority } = createTeamsFileAuthority({ runtimeGeneration, credentialFingerprint, runtime, runtimeContexts, localRuntimeEpoch, runtimeVersions, lockCurrentPrincipalAuthorization, lifecycleRuntimeFence, assertFailedRetryReceipt, inboundWakeScope, failedChatRetrySource, isExplicitOperatorPublication, retryCommentIds });async function authorizeRetainedChatSourcePublication(@@ -15429,7 +15428,6 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {async function processPendingChatMaintenance(limit = 25) {return Promise.allSettled([processPendingTelegramMaintenance(limit),- processFailedChatRunRetries(limit),]);}@@ -15440,7 +15438,8 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {// so a crashed processing claim is quarantined before the delivery worker// could otherwise replay it.const actionRecovery = onlyDeliveryId || !maintenance ? null : Promise.allSettled([- processPendingChatMaintenance(limit), processPendingGitHubWebhookIngress(limit),+ processPendingChatMaintenance(limit), processFailedChatRunRetries(limit),+ processPendingGitHubWebhookIngress(limit),processPendingProviderEffects(limit), processPendingSlackBoardMessages(limit),processPendingSlackTaskStarts(limit), processPendingReceiptReactions(limit),processPendingSlackSessionStops(limit), slackRegistration.processPendingVerificationMessages(limit),diff --git a/server/src/services/chat-channels/failed-run-retry.ts b/server/src/services/chat-channels/failed-run-retry.tsindex c0b8d43988..ae041ef925 100644--- a/server/src/services/chat-channels/failed-run-retry.ts+++ b/server/src/services/chat-channels/failed-run-retry.ts@@ -1,5 +1,6 @@import { continuingTeamsFilePublication, type ContinuingTeamsFileTransfer } from "./microsoft-teams/file-transfer-source.js";-import { notifyChatPublicationWork } from "../chat-work-notifications.js";+import { notifyChatActionWork, notifyChatPublicationWork } from "../chat-work-notifications.js";+import { FAILED_RUN_RETRY_STALE_MS, dueChatAction } from "../chat-action-work.js";import {createHash,@@ -13,7 +14,6 @@ import {isNotNull,isNull,like,- lte,ne,notInArray,or,@@ -104,7 +104,6 @@ type ChatFailedRunRetryOptions = {lifecycleRuntimeFence: (delivery: DeliveryRow) => RetryRuntimeFence | null;isShuttingDown: () => boolean;wakeup: IssueAssignmentWakeupDeps["wakeup"];- staleProcessingMs: number;};export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOptions) {@@ -115,7 +114,6 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOptilifecycleRuntimeFence,isShuttingDown,wakeup,- staleProcessingMs: DELIVERY_PROCESSING_STALE_MS,} = options;@@ -1265,6 +1263,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti.limit(1);if (busy || !input.initiatedByUserId) throw failedChatRetryDenied();const now = new Date();+ await notifyChatActionWork(tx, "failed_run_retry");const [intent] = await tx.insert(chatActions).values({@@ -1508,36 +1507,42 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti!["issued", "processing"].includes(candidate.status) ||(candidate.status === "processing" &&candidate.updatedAt.getTime() >- now.getTime() - DELIVERY_PROCESSING_STALE_MS) ||+ now.getTime() - FAILED_RUN_RETRY_STALE_MS) ||(typeof candidate.result?.retryAt === "string" &&Date.parse(candidate.result.retryAt) > now.getTime()))return failedChatRetryReceipt(candidate);- const [claimed] = await db- .update(chatActions)- .set({ status: "processing", updatedAt: now })- .where(- and(- eq(chatActions.id, candidate.id),- eq(chatActions.status, candidate.status),- eq(chatActions.updatedAt, candidate.updatedAt),- ),- )- .returning();- if (!claimed) return failedChatRetryReceipt(candidate);- const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;- const settle = async (status: string, result: Record<string, unknown>) => {- const [current] = await db+ const [claimed] = await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "failed_run_retry");+ return tx.update(chatActions)- .set({ status, result, updatedAt: new Date() })+ .set({ status: "processing", updatedAt: now }).where(and(- eq(chatActions.id, claimed.id),- eq(chatActions.status, "processing"),- eq(chatActions.updatedAt, now),+ eq(chatActions.id, candidate.id),+ eq(chatActions.status, candidate.status),+ eq(chatActions.updatedAt, candidate.updatedAt),),).returning();+ });+ if (!claimed) return failedChatRetryReceipt(candidate);+ const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;+ const settle = async (status: string, result: Record<string, unknown>) => {+ const [current] = await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "failed_run_retry");+ return tx+ .update(chatActions)+ .set({ status, result, updatedAt: new Date() })+ .where(+ and(+ eq(chatActions.id, claimed.id),+ eq(chatActions.status, "processing"),+ eq(chatActions.updatedAt, now),+ ),+ )+ .returning();+ });return failedChatRetryReceipt(current ?? claimed);};try {@@ -1669,26 +1674,10 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOptiasync function processFailedChatRunRetries(limit = 20) {if (isShuttingDown()) return 0;- const now = new Date();const candidates = await db.select({ id: chatActions.id }).from(chatActions)- .where(- and(- eq(chatActions.kind, "failed_run_retry"),- or(- eq(chatActions.status, "issued"),- and(- eq(chatActions.status, "processing"),- lte(- chatActions.updatedAt,- new Date(now.getTime() - DELIVERY_PROCESSING_STALE_MS),- ),- ),- ),- sql`(${chatActions.result}->>'retryAt' is null or (${chatActions.result}->>'retryAt')::timestamptz <= ${now.toISOString()}::timestamptz)`,- ),- )+ .where(dueChatAction("failed_run_retry")).orderBy(asc(chatActions.createdAt), asc(chatActions.id)).limit(Math.max(1, Math.min(50, limit)));for (const candidate of candidates)diff --git a/server/src/services/chat-work-notifications.ts b/server/src/services/chat-work-notifications.tsindex f76ac13c72..16e2ac7513 100644--- a/server/src/services/chat-work-notifications.ts+++ b/server/src/services/chat-work-notifications.ts@@ -13,6 +13,7 @@ export async function notifyChatPublicationWork(tx: object): Promise<void> {export const CHAT_ACTION_QUEUES = {provider_effect: DELIVERY_QUEUES.chatProviderEffects,github_webhook_ingress: DELIVERY_QUEUES.chatGitHubIngress,+ failed_run_retry: DELIVERY_QUEUES.chatFailedRunRetries,slack_session_sync: DELIVERY_QUEUES.chatSessionSyncs,slack_board_message: DELIVERY_QUEUES.chatBoardMessages,slash_task_start: DELIVERY_QUEUES.chatTaskStarts,diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.tsindex fdee30932f..ab4ba5fe25 100644--- a/server/src/services/delivery-work-notifications.ts+++ b/server/src/services/delivery-work-notifications.ts@@ -8,6 +8,7 @@ export const DELIVERY_QUEUES = {chatSessionStops: "chat-session-stops",chatProviderEffects: "chat-provider-effects",chatGitHubIngress: "chat-github-ingress",+ chatFailedRunRetries: "chat-failed-run-retries",chatSessionSyncs: "chat-session-syncs",chatVerification: "chat-verification",chatInbound: "chat-inbound",