The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
perf(chat): wake action queues from commits and deadlines (#15808)
Dottab1ebe3de3c17
diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.mdindex fe1b460ff8..d818caa41c 100644--- a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md+++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md@@ -422,7 +422,19 @@ Longer polling intervals alone do not complete an item.eligibility predicate so paused endpoints, unknown outcomes, blocked followers,and receipts owned by a live sender do not cause empty scans. Completion wakesrefill free endpoint slots without waiting for slow peers. No new tables.- Remaining **1s** maintenance: provider runtimes, provider-specific action+ Five additional queues now use independent commit/deadline workers:+ Slack Board messages, slash-command task starts, setup verification messages,+ receipt reactions, and Slack session stops. Their producers and foreground+ claim/retry/settlement writes signal inside the owning transaction. The same+ eligibility predicates drive dispatch and deadline selection. Paused task+ starts/Board messages and uncertain sends have no recurring deadline.+ Verification waits for account linking, welcome completion, and a verified+ callback; those state changes wake it. Failed confirmed task admission saves+ a retry deadline so its commit cannot cause an immediate retry loop.+ Startup restores existing rows, and empty workers disarm. No new tables.+ Tests cover an idle hour, real producer notifications, PostgreSQL deadline+ selection, stale claims, outer-transaction wakes, and callback-driven setup.+ Remaining **1s** maintenance: provider runtimes, other provider actionoutboxes, run milestone/wake-notice projection, GitHub operation recovery,Teams protocol maintenance, and Slack session sync. GitHub pre-ingress recoverymust still inspect provider history: a lost callback has no local queue row.diff --git a/server/src/__tests__/app-private-hostname-gate.test.ts b/server/src/__tests__/app-private-hostname-gate.test.tsindex 11cb68ea39..61893417df 100644--- a/server/src/__tests__/app-private-hostname-gate.test.ts+++ b/server/src/__tests__/app-private-hostname-gate.test.ts@@ -93,6 +93,7 @@ describe("createChatReconciliationCoordinator", () => {it("wires bounded publication refill to commit workers and isolates periodic maintenance", () => {const source = readFileSync(new URL("../app.ts", import.meta.url), "utf8");expect(source).toContain("registerChatDeliveryWork(deliveryWork, chatChannels,");+ expect(source).toContain("registerChatActionWork(deliveryWork, chatChannels,");const maintenance = source.slice(source.indexOf("const reconcileChatPublicationMaintenance ="),source.indexOf("const chatReconciliation ="),diff --git a/server/src/__tests__/chat-action-work.test.ts b/server/src/__tests__/chat-action-work.test.tsnew file mode 100644index 0000000000..af541a60c0--- /dev/null+++ b/server/src/__tests__/chat-action-work.test.ts@@ -0,0 +1,109 @@+import { afterEach, describe, expect, it, vi } from "vitest";+import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";+import { registerChatActionWork } from "../services/chat-action-work.js";+import { CHAT_ACTION_QUEUES, notifyChatActionWork, notifyChatEndpointWork, notifyChatVerificationWork } from "../services/chat-work-notifications.js";++const coordinators: ReturnType<typeof createDeliveryWorkCoordinator>[] = [];+afterEach(async () => {+ for (const coordinator of coordinators.splice(0)) await coordinator.stop();+ vi.useRealTimers();+});+function setup() {+ vi.useFakeTimers();+ const execute = vi.fn(async () => [{ xid: "42", status: "committed" }]);+ const owner = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise<any>) {+ return callback({ execute });+ } });+ const enabled = vi.fn(() => true), onError = vi.fn();+ const coordinator = createDeliveryWorkCoordinator({ owner, canRun: enabled, canReconcile: enabled, onError });+ coordinators.push(coordinator);+ const service = {+ processPendingSlackBoardMessages: vi.fn(async () => {}),+ processPendingSlackTaskStarts: vi.fn(async () => {}),+ processPendingReceiptReactions: vi.fn(async () => {}),+ processPendingSlackSessionStops: vi.fn(async () => {}),+ processPendingVerificationMessages: vi.fn(async () => {}),+ nextChatActionAt: vi.fn(async (): Promise<number | null> => null),+ nextVerificationMessageAt: vi.fn(async (): Promise<number | null> => null),+ };+ const runs = [service.processPendingSlackBoardMessages, service.processPendingSlackTaskStarts,+ service.processPendingReceiptReactions, service.processPendingSlackSessionStops, service.processPendingVerificationMessages];+ return { owner, service, runs, coordinator, enabled, onError, start: () => registerChatActionWork(coordinator, service, enabled) };+}+describe("chat action scheduling", () => {+ it("does one startup scan per lane and no more SQL or timers during an idle hour", async () => {+ const s = setup();+ await s.start().ready;+ await vi.advanceTimersByTimeAsync(3_600_000);+ for (const run of s.runs) expect(run).toHaveBeenCalledTimes(1);+ expect(s.service.nextChatActionAt).toHaveBeenCalledTimes(4);+ expect(s.service.nextVerificationMessageAt).toHaveBeenCalledTimes(1);+ expect(s.coordinator.nextWakeAt()).toBeNull();+ expect(vi.getTimerCount()).toBe(0);+ });+ it("wakes each lane after its write settles and wakes parked work when an endpoint changes", async () => {+ const s = setup();+ await s.start().ready;+ for (const kind of Object.keys(CHAT_ACTION_QUEUES) as (keyof typeof CHAT_ACTION_QUEUES)[]) {+ await s.owner.transaction(async tx => {+ await notifyChatActionWork(tx, kind);+ expect(s.runs.every(run => run.mock.calls.length === 1)).toBe(true);+ });+ }+ await s.owner.transaction(tx => notifyChatVerificationWork(tx));+ await vi.advanceTimersByTimeAsync(1);+ for (const run of s.runs) expect(run).toHaveBeenCalledTimes(2);+ await s.owner.transaction(tx => notifyChatEndpointWork(tx));+ await vi.advanceTimersByTimeAsync(1);+ for (const run of s.runs) expect(run).toHaveBeenCalledTimes(3);+ expect(vi.getTimerCount()).toBe(0);+ });+ it("restores durable deadlines and reschedules when a foreground send moves a retry earlier", async () => {+ const s = setup();+ let due: number | null = Date.now() + 120_000;+ s.service.nextChatActionAt.mockImplementation(async () => due);+ await s.start().ready;+ expect(s.coordinator.nextWakeAt()).toBe(due);+ await vi.advanceTimersByTimeAsync(1000);+ due = Date.now() + 30_000;+ await s.owner.transaction(tx => notifyChatActionWork(tx, "receipt_reaction"));+ await vi.advanceTimersByTimeAsync(1);+ expect(s.coordinator.nextWakeAt()).toBe(due);+ expect(s.service.processPendingReceiptReactions).toHaveBeenCalledTimes(2);+ due = null;+ await vi.advanceTimersByTimeAsync(30_000);+ expect(s.service.processPendingReceiptReactions).toHaveBeenCalledTimes(3);+ await vi.advanceTimersByTimeAsync(120_000);+ expect(s.coordinator.nextWakeAt()).toBeNull();+ });+ it("keeps lanes independent and retains a commit received during a drain", async () => {+ const s = setup();+ let release!: () => void;+ s.service.processPendingSlackBoardMessages.mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));+ const work = s.start();+ await vi.advanceTimersByTimeAsync(1);+ await s.owner.transaction(tx => notifyChatActionWork(tx, "slack_board_message"));+ await s.owner.transaction(tx => notifyChatActionWork(tx, "slack_session_stop"));+ await vi.advanceTimersByTimeAsync(1);+ expect(s.service.processPendingSlackSessionStops).toHaveBeenCalledTimes(2);+ release(); await work.ready;+ await vi.advanceTimersByTimeAsync(1);+ expect(s.service.processPendingSlackBoardMessages).toHaveBeenCalledTimes(2);+ });+ it("retries failed recovery and does no dispatch or deadline queries during standby", async () => {+ const s = setup();+ s.enabled.mockReturnValue(false);+ await s.start().ready;+ await vi.advanceTimersByTimeAsync(2000);+ expect(s.service.nextChatActionAt).not.toHaveBeenCalled();+ for (const run of s.runs) expect(run).not.toHaveBeenCalled();+ s.enabled.mockReturnValue(true);+ s.service.processPendingSlackTaskStarts.mockRejectedValueOnce(new Error("DB unavailable"));+ await vi.advanceTimersByTimeAsync(1000);+ expect(s.onError).toHaveBeenCalledTimes(1);+ await vi.advanceTimersByTimeAsync(1000);+ expect(s.service.processPendingSlackTaskStarts).toHaveBeenCalledTimes(2);+ expect(vi.getTimerCount()).toBe(0);+ });+});diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.tsindex 5c15c718f6..118b47a8a8 100644--- a/server/src/__tests__/chat-channels.integration.test.ts+++ b/server/src/__tests__/chat-channels.integration.test.ts@@ -1,6 +1,7 @@+import { registerChatActionWork, dueChatAction } from "../services/chat-action-work.js";import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";import { registerChatDeliveryWork } from "../services/chat-delivery-work.js";-import { notifyChatPublicationWork } from "../services/chat-work-notifications.js";+import { notifyChatPublicationWork, notifyChatActionWork, CHAT_ACTION_QUEUES } from "../services/chat-work-notifications.js";import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js";import { aiConnectionService } from "../services/ai-connections.js";import { fastResponseService } from "../services/fast-responses.js";@@ -4492,7 +4493,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {const delivery = randomUUID();const send = () => f.service.handleWebhook(f.endpoint.publicId, "github", signedGitHubWebhookRequest({ event, delivery, payload, webhookSecret: f.webhookSecret }));expect((await send()).status).toBeLessThan(300);- await expect.poll(async () => (await db.select().from(chatConversations).where(eq(chatConversations.endpointId, f.endpoint.id))).length).toBe(1);+ // Webhook acknowledgement precedes the asynchronous admission transaction.+ await expect.poll(async () => (await db.select().from(chatConversations).where(eq(chatConversations.endpointId, f.endpoint.id))).length, { timeout: 10_000 }).toBe(1);await send();const [conversation] = await db.select().from(chatConversations).where(eq(chatConversations.endpointId, f.endpoint.id));expect(conversation.externalThreadId).toBe(`github:PaperclipAI/Paperclip:${event === "issues" || event === "issue_comment" ? "issue:" : ""}83${event === "pull_request_review_comment" ? ":rc:83000" : ""}`);@@ -6553,6 +6555,27 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {await expect(restarted.slackRegistration.install(f.endpoint.id, actor)).resolves.toHaveProperty("authorizationUrl");expect((await restarted.get(f.endpoint.id)).setup.webhookVerifiedAt).toBeFalsy();});+ it("wakes verification from a signed callback with no maintenance polling", async () => {+ const f = await fixture({ scheduleDeferredWork: () => undefined });+ await f.create(); await f.service.slackRegistration.complete(await f.install(), "code", null, actor);+ const onError = vi.fn();+ const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });+ try {+ await registerChatActionWork(coordinator, f.service, () => true).ready;+ await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 10_000 });+ expect(await f.service.nextVerificationMessageAt()).toBeNull();+ await f.service.handleWebhook(f.endpoint.publicId, "slack", signedSlackWebhookRequest({+ url: f.endpoint.setup.webhookUrl!, contentType: "application/json", signingSecret,+ body: JSON.stringify({ type: "url_verification", challenge: "scheduler" }),+ }));+ await vi.waitFor(async () => {+ expect((await f.service.get(f.endpoint.id)).setup.slackAccount?.verificationStatus).toBe("sent");+ expect(coordinator.nextWakeAt()).toBeNull();+ }, { timeout: 10_000 });+ expect(f.provider.mock.calls.filter(([url]) => String(url).endsWith("chat.postMessage"))).toHaveLength(2);+ expect(onError).not.toHaveBeenCalled();+ } finally { await coordinator.stop(); }+ });it("acknowledges verification before sending one DM across concurrent callbacks and restart recovery", async () => {const f = await fixture({ scheduleDeferredWork: () => undefined });await f.create(); await f.service.slackRegistration.complete(await f.install(), "code", null, actor);@@ -6979,6 +7002,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {await instanceSettingsService(db).updateExperimental({ enableChatConnectors: true });const fixture = await seedCompany();const { callbacks, endpoint, runtime, service, wakeup } = await configuredSlackEndpoint(fixture);+ const workWake = vi.fn();+ const stopWorkObserver = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatBoardMessages, workWake);+ onTestFinished(() => { stopWorkObserver(); expect(workWake).toHaveBeenCalled(); });const channel = makeThread({ channelId: "CBOARD", id: "slack:CBOARD:8000.1", name: "board" });await deliverMessage({ callbacks, endpointId: endpoint.id, thread: channel.thread,message: makeMessage({ id: "8000.1", text: "@maya start here", mentioned: true }), trigger: "mention" });@@ -7013,6 +7039,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {"**Owner User (via Paperclip)**\n\nPlease make the plan", "The plan is ready.",]);// The explicit Board composer uses the same mirror and a durable wakeup.+ workWake.mockClear();const key = randomUUID();await service.publishBoardMessage(endpoint.id, conversation.id, "Now assign the follow-ups", key, "owner-user");await service.publishBoardMessage(endpoint.id, conversation.id, "Now assign the follow-ups", key, "owner-user");@@ -7025,9 +7052,14 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {wakeup.mockRejectedValueOnce(new Error("temporary scheduler failure"));await service.publishBoardMessage(endpoint.id, conversation.id, "Retry this work", randomUUID(), "owner-user");expect(await db.select().from(chatActions).where(and(eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_board_message"), eq(chatActions.status, "received")))).toHaveLength(1);- await service.processPendingDeliveries();+ const recovery = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });+ try {+ await registerChatActionWork(recovery, service, () => true).ready;+ await vi.waitFor(async () => {+ expect(await db.select().from(chatActions).where(and(eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_board_message"), eq(chatActions.status, "received")))).toHaveLength(0);+ });+ } finally { await recovery.stop(); }await service.processPendingPublications();- expect(await db.select().from(chatActions).where(and(eq(chatActions.endpointId, endpoint.id), eq(chatActions.kind, "slack_board_message"), eq(chatActions.status, "received")))).toHaveLength(0);expect(runtime.endpoints.get(endpoint.id)!.posts.filter(post => post.text.includes("Retry this work"))).toHaveLength(1);});@@ -18376,6 +18408,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {});const { callbacks, endpoint, runtime, service } =await configuredSlackEndpoint(fixture, { cancelRun });+ const workWake = vi.fn();+ const stopWorkObserver = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatSessionStops, workWake);+ onTestFinished(() => { stopWorkObserver(); expect(workWake).toHaveBeenCalled(); });const externalUserId = "USTOPPER1";const channelId = "CSTOPSESSION1";const threadTs = `${Math.floor(Date.now() / 1_000) - 5}.100000`;@@ -18482,6 +18517,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {},});const webhookUrl = `https://paperclip.example/api/chat-webhooks/${endpoint.publicId}/slack`;+ workWake.mockClear();const first = await service.handleWebhook(endpoint.publicId,"slack",@@ -27664,6 +27700,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {const fixture = await seedCompany();const { callbacks, endpoint, runtime, service } =await configuredSlackEndpoint(fixture);+ const workWake = vi.fn();+ const stopWorkObserver = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatReceiptReactions, workWake);+ onTestFinished(() => { stopWorkObserver(); expect(workWake).toHaveBeenCalled(); });const endpointRuntime = runtime.endpoints.get(endpoint.id);if (!endpointRuntime) throw new Error("Expected Slack runtime");endpointRuntime.reactionErrors.push(@@ -32853,6 +32892,101 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {return { ...fixture, storage, runtime, service, lanes, enqueue, cleanup };}+ it("restores action retry and stale-claim deadlines and parks paused or uncertain work", async () => {+ const f = await publicationLaneFixture(1);+ const lane = f.lanes[0]!;+ const at = new Date();+ const later = new Date(at.getTime() + 120_000);+ try {+ for (const kind of Object.keys(CHAT_ACTION_QUEUES) as (keyof typeof CHAT_ACTION_QUEUES)[]) {+ const [action] = await db.insert(chatActions).values({ companyId: f.companyId, endpointId: lane.endpoint.id,+ kind, providerActionId: randomUUID(), status: "received", updatedAt: at }).returning();+ const set = (status: string, result: Record<string, unknown> = {}) => db.update(chatActions)+ .set({ status, result, updatedAt: at }).where(eq(chatActions.id, action!.id));+ const due = async () => db.select({ id: chatActions.id }).from(chatActions)+ .leftJoin(chatEndpoints, eq(chatEndpoints.id, chatActions.endpointId))+ .where(and(eq(chatActions.id, action!.id), dueChatAction(kind))).then(rows => rows.length > 0);+ if (kind === "slack_board_message") {+ expect(await due()).toBe(true);+ await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, lane.endpoint.id));+ expect(await f.service.nextChatActionAt(kind)).toBeNull();+ expect(await due()).toBe(false);+ await db.update(chatEndpoints).set({ status: "active" }).where(eq(chatEndpoints.id, lane.endpoint.id));+ } else if (kind === "slash_task_start") {+ for (const [status, delay] of [["validating", 60_000], ["resolving", 300_000], ["admitting", 60_000]] as const) {+ await set(status);+ expect(await f.service.nextChatActionAt(kind)).toBe(at.getTime() + delay);+ expect(await due()).toBe(false);+ }+ for (const status of ["queued", "provider_confirmed"]) {+ await set(status, { retryAt: later.toISOString() });+ expect(await f.service.nextChatActionAt(kind)).toBe(later.getTime());+ expect(await due()).toBe(false);+ await db.update(chatEndpoints).set({ status: "attention" }).where(eq(chatEndpoints.id, lane.endpoint.id));+ expect(await f.service.nextChatActionAt(kind)).toBeNull();+ await db.update(chatEndpoints).set({ status: "active" }).where(eq(chatEndpoints.id, lane.endpoint.id));+ }+ await set("delivery_unknown");+ expect(await f.service.nextChatActionAt(kind)).toBeNull();+ expect(await due()).toBe(false);+ } else {+ expect(await due()).toBe(true);+ await set("processing");+ expect(await f.service.nextChatActionAt(kind)).toBe(at.getTime() + 60_000);+ expect(await due()).toBe(false);+ await set("failed", { retryable: true, retryAt: later.toISOString() });+ expect(await f.service.nextChatActionAt(kind)).toBe(later.getTime());+ expect(await due()).toBe(false);+ await set("failed", { retryable: false });+ expect(await f.service.nextChatActionAt(kind)).toBeNull();+ }+ await set("processed");+ expect(await f.service.nextChatActionAt(kind)).toBeNull();+ }+ } finally { await f.cleanup(); }+ });++ it("recovers action rows on startup, wakes after outer commit, and leaves empty queues quiet", async () => {+ const f = await publicationLaneFixture(1);+ const lane = f.lanes[0]!;+ const onError = vi.fn();+ const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError });+ const action = (kind: "receipt_reaction" | "slack_session_stop") => ({ companyId: f.companyId,+ endpointId: lane.endpoint.id, kind, providerActionId: randomUUID(), status: "processing",+ updatedAt: new Date(Date.now() - 120_000), payload: {} });+ try {+ const [stale] = await db.insert(chatActions).values(action("receipt_reaction")).returning();+ await registerChatActionWork(coordinator, f.service, () => true).ready;+ await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 10_000 });+ expect((await db.select().from(chatActions).where(eq(chatActions.id, stale!.id)))[0]?.status).toBe("failed");+ const scans = vi.spyOn(db, "select");+ try {+ await new Promise(resolve => setTimeout(resolve, 1200));+ expect(scans).not.toHaveBeenCalled();+ } finally { scans.mockRestore(); }+ let release!: () => void, inserted!: () => void;+ const gate = new Promise<void>(resolve => { release = resolve; });+ const staged = new Promise<void>(resolve => { inserted = resolve; });+ const writing = db.transaction(async tx => {+ await tx.transaction(async nested => {+ await notifyChatActionWork(nested, "slack_session_stop");+ await nested.insert(chatActions).values(action("slack_session_stop"));+ });+ inserted(); await gate;+ });+ await staged;+ expect(coordinator.nextWakeAt()).not.toBeNull();+ release(); await writing;+ await vi.waitFor(async () => {+ const rows = await db.select().from(chatActions).where(and(eq(chatActions.companyId, f.companyId), eq(chatActions.kind, "slack_session_stop")));+ expect(rows[0]?.status).toBe("failed");+ expect(coordinator.nextWakeAt()).toBeNull();+ }, { timeout: 10_000 });+ expect(lane.providerRuntime.posts).toHaveLength(0);+ expect(onError).not.toHaveBeenCalled();+ } finally { await coordinator.stop(); await f.cleanup(); }+ });+it("restores chat queue deadlines without polling blocked followers or receipt owners", async () => {const f = await publicationLaneFixture(1);const { service, lanes, enqueue } = f;@@ -53054,6 +53188,9 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {const fixture = await seedCompany();const { callbacks, endpoint, runtime, service } =await configuredSlackEndpoint(fixture);+ const workWake = vi.fn();+ const stopWorkObserver = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatTaskStarts, workWake);+ onTestFinished(() => { stopWorkObserver(); expect(workWake).toHaveBeenCalled(); });const command = endpoint.setup.command;if (!command) throw new Error("Slack endpoint did not expose its command");if (!callbacks.onSlashCommand) {diff --git a/server/src/app.ts b/server/src/app.tsindex 7fcff18b2d..003d587c55 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -13,6 +13,7 @@ import { customerSuccessRoutes } from "./routes/customer-success.js";import { cloudWarmStandbyMiddleware } from "./middleware/cloud-warm-standby.js";import type { CloudWarmStandby } from "./services/cloud-warm-standby.js";import { browserUseRoutes } from "./routes/browser-use.js";+import { registerChatActionWork } from "./services/chat-action-work.js";import { registerChatDeliveryWork } from "./services/chat-delivery-work.js";import { registerBrowserUseCleanup } from "./services/browser-use-work.js";import { browserUseService } from "./services/browser-use.js";@@ -1252,6 +1253,7 @@ export async function createApp(});}registerChatDeliveryWork(deliveryWork, chatChannels, () => !isIdleTaskDrainActive());+ registerChatActionWork(deliveryWork, chatChannels, () => !isIdleTaskDrainActive());emailChannels.start();const reconcileChatPublicationMaintenance = async () => {await chatChannels.processPublicationMaintenance();diff --git a/server/src/services/chat-action-work.ts b/server/src/services/chat-action-work.tsnew file mode 100644index 0000000000..4299395054--- /dev/null+++ b/server/src/services/chat-action-work.ts@@ -0,0 +1,85 @@+import { and, eq, inArray, lte, or, sql } from "drizzle-orm";+import { chatActions, chatEndpoints, type Db } from "@paperclipai/db";+import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";+import { CHAT_ACTION_QUEUES, type ScheduledChatAction } from "./chat-work-notifications.js";+import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";+import { registerChatQueueWork } from "./chat-delivery-work.js";++export const SLACK_COMMAND_POST_STALE_MS = 60_000;+export const SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS = 5 * 60_000;+export const SLACK_COMMAND_ADMISSION_STALE_MS = 60_000;+export const PROVIDER_EFFECT_STALE_MS = 60_000;++/** The dispatch predicate and scheduler use the same eligibility and deadlines.+ * Zero means ready now; null means terminal or waiting for an external change. */+export function chatActionDeadline(kind: ScheduledChatAction) {+ const updated = sql`extract(epoch from ${chatActions.updatedAt}) * 1000`;+ const retry = sql`coalesce(extract(epoch from (${chatActions.result}->>'retryAt')::timestamptz) * 1000, 0)`;+ if (kind === "slack_board_message") return sql<number>`case+ when ${chatActions.status} = 'received' and ${chatEndpoints.status} not in ('paused', 'attention') then 0::numeric end`;+ if (kind === "slash_task_start") return sql<number>`case+ when ${chatActions.status} = 'queued' and (${chatEndpoints.id} is null or ${chatEndpoints.status} in ('verifying', 'active', 'archived')) then ${retry}+ when ${chatActions.status} = 'provider_confirmed' and (${chatEndpoints.id} is null or ${chatEndpoints.status} in ('verifying', 'active', 'archived')) then ${retry}+ when ${chatActions.status} = 'admitting' then ${updated} + ${SLACK_COMMAND_ADMISSION_STALE_MS}+ when ${chatActions.status} = 'resolving' then ${updated} + ${SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS}+ when ${chatActions.status} = 'validating' then ${updated} + ${SLACK_COMMAND_POST_STALE_MS} end`;+ return sql<number>`case+ when ${chatActions.status} = 'received' then 0+ when ${chatActions.status} = 'processing' then ${updated} + ${PROVIDER_EFFECT_STALE_MS}+ when ${chatActions.status} = 'failed' and ${chatActions.result}->>'retryable' = 'true' then ${retry} end`;+}++function pendingChatAction(kind: ScheduledChatAction) {+ // Filter history before the endpoint join and aggregate. Failed reactions+ // and stops only remain live when the saved result permits another attempt.+ const status = kind === "slack_board_message"+ ? eq(chatActions.status, "received")+ : kind === "slash_task_start"+ ? inArray(chatActions.status, ["queued", "provider_confirmed", "admitting", "resolving", "validating"])+ : or(inArray(chatActions.status, ["received", "processing"]),+ and(eq(chatActions.status, "failed"), sql`${chatActions.result}->>'retryable' = 'true'`));+ return and(eq(chatActions.kind, kind), status);+}++export function dueChatAction(kind: ScheduledChatAction, now = Date.now()) {+ return and(pendingChatAction(kind), lte(chatActionDeadline(kind), now));+}++export async function nextChatActionAt(db: Db, kind: ScheduledChatAction): Promise<number | null> {+ const [row] = await db.select({ at: sql<string | null>`min(${chatActionDeadline(kind)})` })+ .from(chatActions)+ .leftJoin(chatEndpoints, and(eq(chatEndpoints.id, chatActions.endpointId), eq(chatEndpoints.companyId, chatActions.companyId)))+ .where(pendingChatAction(kind));+ return row?.at == null ? null : Number(row.at);+}++type ChatActionService = {+ processPendingSlackBoardMessages(): Promise<unknown>;+ processPendingSlackTaskStarts(): Promise<unknown>;+ processPendingReceiptReactions(): Promise<unknown>;+ processPendingSlackSessionStops(): Promise<unknown>;+ processPendingVerificationMessages(): Promise<unknown>;+ nextChatActionAt(kind: ScheduledChatAction): Promise<number | null>;+ nextVerificationMessageAt(): Promise<number | null>;+};++export function registerChatActionWork(+ coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,+ service: ChatActionService,+ canRun: () => boolean,+) {+ const actions = [+ { kind: "slack_board_message", run: service.processPendingSlackBoardMessages },+ { kind: "slash_task_start", run: service.processPendingSlackTaskStarts },+ { kind: "receipt_reaction", run: service.processPendingReceiptReactions },+ { kind: "slack_session_stop", run: service.processPendingSlackSessionStops },+ ] as const;+ const workers = actions.map(({ kind, run }) => registerChatQueueWork(coordinator, {+ queue: CHAT_ACTION_QUEUES[kind], run, next: () => service.nextChatActionAt(kind),+ }, canRun));+ workers.push(registerChatQueueWork(coordinator, {+ queue: DELIVERY_QUEUES.chatVerification,+ run: service.processPendingVerificationMessages, next: service.nextVerificationMessageAt,+ }, canRun));+ return { ready: Promise.all(workers.map(worker => worker.ready)) };+}diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.tsindex d6b1d41b9c..d81f4ea874 100644--- a/server/src/services/chat-channels.ts+++ b/server/src/services/chat-channels.ts@@ -1,4 +1,6 @@-import { notifyChatDeliveryWork, notifyChatPublicationWork, notifyChatEndpointWork } from "./chat-work-notifications.js";+import { dueChatAction, nextChatActionAt, SLACK_COMMAND_POST_STALE_MS, SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS, SLACK_COMMAND_ADMISSION_STALE_MS, PROVIDER_EFFECT_STALE_MS } from "./chat-action-work.js";+import type { ScheduledChatAction } from "./chat-work-notifications.js";+import { notifyChatDeliveryWork, notifyChatPublicationWork, notifyChatEndpointWork, notifyChatActionWork, notifyChatVerificationWork } from "./chat-work-notifications.js";import { syncSpekoVoiceTools } from "./voice/speko-agent-tools.js";import { configureSpekoSessionTools } from "./voice/speko-tool-setup.js";import { voiceSessionService } from "./voice/voice-session-service.js";@@ -862,10 +864,6 @@ const REACTION_LINK_MAX_AGE_MS = 2 * 60_000;const DISCORD_ROOT_THREAD_CONFIRMATION_DELAY_MS = 5 * 60_000;const DELIVERY_LEASE_TTL_MS = 90_000;const DELIVERY_DRAIN_LIMIT = 100;-const SLACK_COMMAND_POST_STALE_MS = 60_000;-const SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS = 5 * 60_000;-const SLACK_COMMAND_ADMISSION_STALE_MS = 60_000;-const PROVIDER_EFFECT_STALE_MS = 60_000;const SLACK_FILE_RECEIPT_MAX_AGE_MS = 24 * 60 * 60_000;const SLACK_FILE_RECEIPT_MAX_ATTEMPTS = 12;const SLACK_FILE_RECEIPT_STALE_MS = 60_000;@@ -3472,27 +3470,30 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {// endpoint again after a rotation and accidentally blessing an obsolete// adapter with the new generation.const context = input.runtimeContext ?? runtimeContextForRecord(record);- const [inserted] = await db- .insert(chatActions)- .values({- companyId: input.endpoint.companyId,- endpointId: input.endpoint.id,- deliveryId: input.deliveryId,- kind: "receipt_reaction",- providerActionId: `receipt_reaction:${input.deliveryId}`,- payload: {- version: 1,- operation: "add",- threadId: input.thread.id,- messageId: input.message.id,- reaction: "eyes",- runtimeGeneration: context.generation,- credentialFingerprint: context.credentialFingerprint,- } satisfies ReceiptReactionPayload,- status: "received",- })- .onConflictDoNothing()- .returning({ id: chatActions.id });+ const [inserted] = await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "receipt_reaction");+ return tx+ .insert(chatActions)+ .values({+ companyId: input.endpoint.companyId,+ endpointId: input.endpoint.id,+ deliveryId: input.deliveryId,+ kind: "receipt_reaction",+ providerActionId: `receipt_reaction:${input.deliveryId}`,+ payload: {+ version: 1,+ operation: "add",+ threadId: input.thread.id,+ messageId: input.message.id,+ reaction: "eyes",+ runtimeGeneration: context.generation,+ credentialFingerprint: context.credentialFingerprint,+ } satisfies ReceiptReactionPayload,+ status: "received",+ })+ .onConflictDoNothing()+ .returning({ id: chatActions.id });+ });const actionId =inserted?.id ??(await db@@ -3621,17 +3622,20 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {if (!action || ["processed", "cancelled"].includes(action.status)) return;const payload = receiptReactionPayload(action.payload);if (!payload) {- await db- .update(chatActions)- .set({- status: "failed",- result: {- code: "receipt_reaction_payload_invalid",- retryable: false,- },- updatedAt: new Date(),- })- .where(eq(chatActions.id, action.id));+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "receipt_reaction");+ return tx+ .update(chatActions)+ .set({+ status: "failed",+ result: {+ code: "receipt_reaction_payload_invalid",+ retryable: false,+ },+ updatedAt: new Date(),+ })+ .where(eq(chatActions.id, action.id));+ });return;}if (payload.operation === "add" && action.deliveryId) {@@ -3651,6 +3655,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {if (terminalRemoval) {await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "receipt_reaction");await tx.update(chatActions).set({@@ -3680,24 +3685,27 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {// Adding or removing the same reaction is idempotent. A worker// interruption can be retried safely without an ambiguous// duplicate-message boundary.- await db- .update(chatActions)- .set({- status: "failed",- result: {- ...(action.result ?? {}),- retryable: true,- retryAt: new Date().toISOString(),- },- updatedAt: new Date(),- })- .where(- and(- eq(chatActions.id, action.id),- eq(chatActions.status, "processing"),- eq(chatActions.updatedAt, action.updatedAt),- ),- );+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "receipt_reaction");+ return tx+ .update(chatActions)+ .set({+ status: "failed",+ result: {+ ...(action.result ?? {}),+ retryable: true,+ retryAt: new Date().toISOString(),+ },+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(chatActions.id, action.id),+ eq(chatActions.status, "processing"),+ eq(chatActions.updatedAt, action.updatedAt),+ ),+ );+ });action = await db.select().from(chatActions)@@ -3723,6 +3731,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {);const [claimed] = await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "receipt_reaction");const rows = await tx.update(chatActions).set({@@ -3760,6 +3769,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {) {await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "receipt_reaction");await tx.update(chatActions).set({@@ -3853,6 +3863,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {// for an obsolete runtime.await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "receipt_reaction");const processed = await tx.update(chatActions).set({@@ -3898,6 +3909,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {) {await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "receipt_reaction");const processed = await tx.update(chatActions).set({@@ -3957,6 +3969,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {: null;await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "receipt_reaction");const [failed] = await tx.update(chatActions).set({@@ -4013,31 +4026,9 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {limit = 25,onlyActionId?: string,) {- const now = new Date();- const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS);- const actions = await db- .select({ id: chatActions.id })- .from(chatActions)- .where(- and(- eq(chatActions.kind, "receipt_reaction"),- ...(onlyActionId ? [eq(chatActions.id, onlyActionId)] : []),- or(- eq(chatActions.status, "received"),- and(- eq(chatActions.status, "processing"),- lte(chatActions.updatedAt, staleBefore),- ),- and(- eq(chatActions.status, "failed"),- sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'true'`,- sql`(${chatActions.result}->>'retryAt' is null or (${chatActions.result}->>'retryAt')::timestamptz <= ${now.toISOString()}::timestamptz)`,- ),- ),- ),- )- .orderBy(asc(chatActions.createdAt))- .limit(limit);+ const actions = await db.select({ id: chatActions.id }).from(chatActions)+ .where(and(dueChatAction("receipt_reaction"), onlyActionId ? eq(chatActions.id, onlyActionId) : undefined))+ .orderBy(asc(chatActions.createdAt)).limit(limit);for (const action of actions) await processReceiptReaction(action.id);return actions.length;}@@ -22226,6 +22217,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {);const finalized = await db.transaction(async (tx) => {await input.credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "slash_task_start");const [ownedAction] = await tx.update(chatActions).set({@@ -22321,28 +22313,31 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {const staleAdmission = new Date(Date.now() - SLACK_COMMAND_ADMISSION_STALE_MS,);- const [action] = await db- .update(chatActions)- .set({- status: "admitting",- result: sql`coalesce(${chatActions.result}, '{}'::jsonb)- || jsonb_build_object('admissionClaimId', ${admissionClaimId}::text)`,- updatedAt: new Date(),- })- .where(- and(- eq(chatActions.id, actionId),- eq(chatActions.kind, "slash_task_start"),- or(- eq(chatActions.status, "provider_confirmed"),- and(- eq(chatActions.status, "admitting"),- lte(chatActions.updatedAt, staleAdmission),+ const [action] = await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({+ status: "admitting",+ result: sql`coalesce(${chatActions.result}, '{}'::jsonb)+ || jsonb_build_object('admissionClaimId', ${admissionClaimId}::text)`,+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(chatActions.id, actionId),+ eq(chatActions.kind, "slash_task_start"),+ or(+ eq(chatActions.status, "provider_confirmed"),+ and(+ eq(chatActions.status, "admitting"),+ lte(chatActions.updatedAt, staleAdmission),+ ),),),- ),- )- .returning();+ )+ .returning();+ });if (!action) return false;await options.slackTaskAdmissionClaimBarrier?.({actionId: action.id,@@ -22358,18 +22353,21 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {sql`${chatActions.result}->>'admissionClaimId' = ${admissionClaimId}`,);const failAdmission = async (code: string, detail: string) => {- await db- .update(chatActions)- .set({- status: "failed",- result: {- ...(confirmed ?? {}),- code,- redactedError: redactSensitiveText(detail).slice(0, MAX_ERROR_TEXT),- },- updatedAt: new Date(),- })- .where(ownsAdmission());+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({+ status: "failed",+ result: {+ ...(confirmed ?? {}),+ code,+ redactedError: redactSensitiveText(detail).slice(0, MAX_ERROR_TEXT),+ },+ updatedAt: new Date(),+ })+ .where(ownsAdmission());+ });};if (!recovery || !confirmed || !action.principalId) {await failAdmission(@@ -22397,14 +22395,17 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {// The provider-visible send is already confirmed. Wait for an// operator to repair/resume the endpoint, then continue Paperclip-only// admission without sending another Slack message.- await db- .update(chatActions)- .set({- status: "provider_confirmed",- result: confirmed,- updatedAt: new Date(),- })- .where(ownsAdmission());+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({+ status: "provider_confirmed",+ result: confirmed,+ updatedAt: new Date(),+ })+ .where(ownsAdmission());+ });}return true;}@@ -22469,27 +22470,34 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {false,true,);- await db- .update(chatActions)- .set({ status: "processed", result: confirmed, updatedAt: new Date() })- .where(ownsAdmission());+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({ status: "processed", result: confirmed, updatedAt: new Date() })+ .where(ownsAdmission());+ });return true;} catch (error) {// The provider post is already confirmed, so this retry is strictly a// Paperclip admission retry. Preserve the root tuple and let the durable// reconciler resume without ever posting to Slack again.- await db- .update(chatActions)- .set({- status: "provider_confirmed",- result: {- ...confirmed,- code: "slash_task_admission_retry",- redactedError: redactError(error),- },- updatedAt: new Date(),- })- .where(ownsAdmission());+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({+ status: "provider_confirmed",+ result: {+ ...confirmed,+ code: "slash_task_admission_retry",+ retryAt: new Date(Date.now() + 1_000).toISOString(),+ redactedError: redactError(error),+ },+ updatedAt: new Date(),+ })+ .where(ownsAdmission());+ });throw error;}}@@ -22528,6 +22536,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {async (credentialLease) => {await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "slash_task_start");await tx.update(chatActions).set({@@ -22570,6 +22579,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {async (credentialLease) => {await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "slash_task_start");await tx.update(chatActions).set({ status: "queued", updatedAt: new Date() })@@ -22609,21 +22619,24 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {? Math.max(0, action.result.attemptCount): 0) + 1;- const [claimed] = await db- .update(chatActions)- .set({- status: "validating",- result: { attemptCount: attempt },- updatedAt: new Date(),- })- .where(- and(- eq(chatActions.id, action.id),- eq(chatActions.kind, "slash_task_start"),- eq(chatActions.status, action.status),- ),- )- .returning();+ const [claimed] = await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({+ status: "validating",+ result: { attemptCount: attempt },+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(chatActions.id, action.id),+ eq(chatActions.kind, "slash_task_start"),+ eq(chatActions.status, action.status),+ ),+ )+ .returning();+ });if (!claimed) return false;action = claimed;@@ -22634,24 +22647,27 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {!initialRecord ||initialRecord.endpoint.provider !== "slack") {- await db- .update(chatActions)- .set({- status: "failed",- result: {- code: "slash_task_provider_context_missing",- retryable: false,- attemptCount: attempt,- },- updatedAt: new Date(),- })- .where(- and(- eq(chatActions.id, action.id),- eq(chatActions.status, "validating"),- sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`,- ),- );+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({+ status: "failed",+ result: {+ code: "slash_task_provider_context_missing",+ retryable: false,+ attemptCount: attempt,+ },+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(chatActions.id, action.id),+ eq(chatActions.status, "validating"),+ sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`,+ ),+ );+ });return true;}let providerPostStarted = false;@@ -22666,53 +22682,59 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {async (credentialLease) => {const record = await endpointRecord(action.endpointId);if (!record || record.endpoint.provider !== "slack") {- await db- .update(chatActions)- .set({- status: "failed",- result: {- code: "slash_task_provider_context_missing",- retryable: false,- attemptCount: attempt,- },- updatedAt: new Date(),- })- .where(- and(- eq(chatActions.id, action.id),- eq(chatActions.status, "validating"),- sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`,- ),- );+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({+ status: "failed",+ result: {+ code: "slash_task_provider_context_missing",+ retryable: false,+ attemptCount: attempt,+ },+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(chatActions.id, action.id),+ eq(chatActions.status, "validating"),+ sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`,+ ),+ );+ });return true;}failureRuntimeFence = runtimeContextForRecord(record);if (!["verifying", "active"].includes(record.endpoint.status)) {const terminallyArchived = record.endpoint.status === "archived";- await db- .update(chatActions)- .set({- status: terminallyArchived ? "cancelled" : "queued",- result: {- code: terminallyArchived- ? "slash_task_endpoint_archived"- : "slash_task_endpoint_waiting",- attemptCount: attempt - 1,- ...(terminallyArchived- ? {}- : {- retryAt: new Date(Date.now() + 5_000).toISOString(),- }),- },- updatedAt: new Date(),- })- .where(- and(- eq(chatActions.id, action.id),- eq(chatActions.status, "validating"),- sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`,- ),- );+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({+ status: terminallyArchived ? "cancelled" : "queued",+ result: {+ code: terminallyArchived+ ? "slash_task_endpoint_archived"+ : "slash_task_endpoint_waiting",+ attemptCount: attempt - 1,+ ...(terminallyArchived+ ? {}+ : {+ retryAt: new Date(Date.now() + 5_000).toISOString(),+ }),+ },+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(chatActions.id, action.id),+ eq(chatActions.status, "validating"),+ sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`,+ ),+ );+ });return false;}@@ -22787,23 +22809,26 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {? record.endpoint.allowDirectMessages: nonDirectDestinationAllowed(record.endpoint, resource);if (!principalAllowed || !destinationAllowed) {- await db- .update(chatActions)- .set({- status: "cancelled",- result: {- code: "slash_task_no_longer_authorized",- attemptCount: attempt,- },- updatedAt: new Date(),- })- .where(- and(- eq(chatActions.id, action.id),- eq(chatActions.status, "validating"),- sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`,- ),- );+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slash_task_start");+ return tx+ .update(chatActions)+ .set({+ status: "cancelled",+ result: {+ code: "slash_task_no_longer_authorized",+ attemptCount: attempt,+ },+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(chatActions.id, action.id),+ eq(chatActions.status, "validating"),+ sql`(${chatActions.result}->>'attemptCount')::int = ${attempt}`,+ ),+ );+ });return true;}@@ -22855,6 +22880,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {!currentAuthorization?.allowed ||!currentDestinationAllowed) {+ await notifyChatActionWork(tx, "slash_task_start");await tx.update(chatActions).set({@@ -22876,6 +22902,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {}failureResource = currentResource;+ await notifyChatActionWork(tx, "slash_task_start");const [transportClaim] = await tx.update(chatActions).set({@@ -22934,6 +22961,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {const threadId = slackTaskStarterThreadId(rawChannelId, starter.id);const [providerConfirmed] = await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "slash_task_start");return tx.update(chatActions).set({@@ -22997,62 +23025,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {}}- async function processPendingSlackTaskStarts(limit: number) {- const now = new Date();- const staleAdmission = new Date(- now.getTime() - SLACK_COMMAND_ADMISSION_STALE_MS,- );- const staleProviderPost = new Date(- now.getTime() - SLACK_COMMAND_EXPLICIT_RETRY_STALE_MS,- );- const staleValidation = new Date(- now.getTime() - SLACK_COMMAND_POST_STALE_MS,- );- const actions = await db- .select({ id: chatActions.id })- .from(chatActions)- .leftJoin(chatEndpoints, eq(chatEndpoints.id, chatActions.endpointId))- .where(- // Provider-confirmed rows need no provider retry and have no due- // timestamp. Park them while an endpoint is repairable but paused- // or in attention; otherwise an old page can starve active work- // forever. Archived and missing endpoints remain eligible so their- // actions can be terminalized.- and(- eq(chatActions.kind, "slash_task_start"),- or(- and(- eq(chatActions.status, "queued"),- sql`(${chatActions.result}->>'retryAt' is null or ${chatActions.result}->>'retryAt' <= ${now.toISOString()})`,- ),- and(- eq(chatActions.status, "provider_confirmed"),- or(- isNull(chatEndpoints.id),- inArray(chatEndpoints.status, [- "verifying",- "active",- "archived",- ]),- ),- ),- and(- eq(chatActions.status, "admitting"),- lte(chatActions.updatedAt, staleAdmission),- ),- and(- eq(chatActions.status, "resolving"),- lte(chatActions.updatedAt, staleProviderPost),- ),- and(- eq(chatActions.status, "validating"),- lte(chatActions.updatedAt, staleValidation),- ),- ),- ),- )- .orderBy(asc(chatActions.createdAt))- .limit(limit);+ async function processPendingSlackTaskStarts(limit = 25) {+ const actions = await db.select({ id: chatActions.id }).from(chatActions)+ .leftJoin(chatEndpoints, and(eq(chatEndpoints.id, chatActions.endpointId), eq(chatEndpoints.companyId, chatActions.companyId)))+ .where(dueChatAction("slash_task_start"))+ .orderBy(asc(chatActions.createdAt)).limit(limit);let processed = 0;for (let offset = 0; offset < actions.length; offset += 4) {const results = await Promise.all(@@ -23770,6 +23747,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {)))return null;+ await notifyChatActionWork(tx, "slash_task_start");return tx.insert(chatActions).values({@@ -26385,6 +26363,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {threadId: event.threadId,userId: event.userId,};+ await notifyChatActionWork(tx, "slack_session_stop");const [inserted] = await tx.insert(chatActions).values({@@ -26453,18 +26432,21 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {: 0;const attempts = previousAttempts + 1;if (!payload) {- await db- .update(chatActions)- .set({- status: "failed",- result: {- attempts,- code: "slack_session_stop_payload_invalid",- retryable: false,- },- updatedAt: now,- })- .where(eq(chatActions.id, action.id));+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slack_session_stop");+ return tx+ .update(chatActions)+ .set({+ status: "failed",+ result: {+ attempts,+ code: "slack_session_stop_payload_invalid",+ retryable: false,+ },+ updatedAt: now,+ })+ .where(eq(chatActions.id, action.id));+ });return;}@@ -26502,6 +26484,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {actorUserId: string | null,targetKind: "run" | "wakeup" | "none",): Promise<StopClaim> => {+ await notifyChatActionWork(tx, "slack_session_stop");await tx.update(chatActions).set({@@ -26741,6 +26724,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {}runId = wakeup?.runId ?? null;if (!runId && wakeup?.status === "claimed") {+ await notifyChatActionWork(tx, "slack_session_stop");await tx.update(chatActions).set({@@ -26809,6 +26793,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {"run",);}+ await notifyChatActionWork(tx, "slack_session_stop");await tx.update(chatActions).set({@@ -26858,6 +26843,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {.then((rows) => rows[0] ?? null);const cancellationWon = authoritativeRun?.status === "cancelled";await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slack_session_stop");const [settled] = await tx.update(chatActions).set({@@ -26903,32 +26889,35 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {});} catch (error) {const retryable = attempts < 5;- await db- .update(chatActions)- .set({- status: "failed",- result: {- attempts,- code: "slack_session_stop_cancellation_failed",- retryable,- ...(retryable- ? {- retryAt: new Date(- Date.now() + Math.min(60_000, 1_000 * 2 ** (attempts - 1)),- ).toISOString(),- }- : {}),- },- updatedAt: new Date(),- })- .where(- and(- eq(chatActions.id, action.id),- eq(chatActions.status, "processing"),- sql`(${chatActions.result}->>'attempts')::int = ${attempts}`,- sql`${chatActions.result}->>'runId' = ${claim.runId}`,- ),- );+ await db.transaction(async (tx) => {+ await notifyChatActionWork(tx, "slack_session_stop");+ return tx+ .update(chatActions)+ .set({+ status: "failed",+ result: {+ attempts,+ code: "slack_session_stop_cancellation_failed",+ retryable,+ ...(retryable+ ? {+ retryAt: new Date(+ Date.now() + Math.min(60_000, 1_000 * 2 ** (attempts - 1)),+ ).toISOString(),+ }+ : {}),+ },+ updatedAt: new Date(),+ })+ .where(+ and(+ eq(chatActions.id, action.id),+ eq(chatActions.status, "processing"),+ sql`(${chatActions.result}->>'attempts')::int = ${attempts}`,+ sql`${chatActions.result}->>'runId' = ${claim.runId}`,+ ),+ );+ });logger.warn({actionId: action.id,@@ -26945,31 +26934,9 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {limit = 25,onlyActionId?: string,) {- const now = new Date();- const staleBefore = new Date(now.getTime() - PROVIDER_EFFECT_STALE_MS);- const actions = await db- .select({ id: chatActions.id })- .from(chatActions)- .where(- and(- eq(chatActions.kind, "slack_session_stop"),- ...(onlyActionId ? [eq(chatActions.id, onlyActionId)] : []),- or(- eq(chatActions.status, "received"),- and(- eq(chatActions.status, "processing"),- lte(chatActions.updatedAt, staleBefore),- ),- and(- eq(chatActions.status, "failed"),- sql`coalesce(${chatActions.result}->>'retryable', 'false') = 'true'`,- sql`(${chatActions.result}->>'retryAt' is null or (${chatActions.result}->>'retryAt')::timestamptz <= ${now.toISOString()}::timestamptz)`,- ),- ),- ),- )- .orderBy(asc(chatActions.createdAt))- .limit(limit);+ const actions = await db.select({ id: chatActions.id }).from(chatActions)+ .where(and(dueChatAction("slack_session_stop"), onlyActionId ? eq(chatActions.id, onlyActionId) : undefined))+ .orderBy(asc(chatActions.createdAt)).limit(limit);for (const action of actions) await processSlackSessionStop(action.id);return actions.length;}@@ -27005,6 +26972,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {const observedAt = new Date();const setup = current.setup as InternalSetupState;const matchesCurrentUrl = slackCallbackMatchesPublicUrl(callback.url, `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${current.publicId}/slack`);+ await notifyChatVerificationWork(tx);await tx.update(chatEndpoints).set({ setup: {...setup,...(matchesCurrentUrl ? { webhookVerifiedAt: setup.webhookVerifiedAt ?? observedAt.toISOString(),@@ -27622,6 +27590,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {}}connectionProved = signingFingerprint !== null;+ if (connectionProved) await notifyChatVerificationWork(tx);await tx.update(chatEndpoints).set({@@ -28045,7 +28014,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {async function processPendingSlackBoardMessages(limit = 25) {const pending = await db.select({ id: chatActions.id, companyId: chatActions.companyId, endpointId: chatActions.endpointId }).from(chatActions).innerJoin(chatEndpoints, and(eq(chatEndpoints.id, chatActions.endpointId), eq(chatEndpoints.companyId, chatActions.companyId)))- .where(and(eq(chatActions.kind, "slack_board_message"), eq(chatActions.status, "received"), notInArray(chatEndpoints.status, ["paused", "attention"])))+ .where(dueChatAction("slack_board_message")).orderBy(asc(chatActions.createdAt)).limit(limit);for (const candidate of pending) {await withSlackBoardLease(db, { ...candidate, actionId: candidate.id }, fetchImpl, async (lease) => {@@ -28057,6 +28026,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {const commentId = String(action.payload.commentId);async function cancelWork(message: string) {await lease.commit(async (tx) => {+ await notifyChatActionWork(tx, "slack_board_message");await tx.update(chatActions).set({ status: "cancelled", result: { code: "slack_board_work_not_authorized", message }, updatedAt: new Date() }).where(eq(chatActions.id, action.id));await notifyChatPublicationWork(tx);await tx.update(chatPublications).set({ state: "cancelled", redactedError: message, nextAttemptAt: null, updatedAt: new Date() }).where(and(@@ -28091,6 +28061,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {await issueService(tx as unknown as Db).update(current.id, { status: "todo", actorUserId: userId });await logActivity(tx as unknown as Db, { companyId: current.companyId, actorType: "user", actorId: userId, action: "issue.updated", entityType: "issue", entityId: current.id, details: { status: "todo", source: "slack_board_message" } });}+ await notifyChatActionWork(tx, "slack_board_message");await tx.update(chatActions).set({ result: { resumeApplied: true }, updatedAt: new Date() }).where(eq(chatActions.id, action.id));}return current;@@ -28105,6 +28076,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {});await lease.commit(async (tx) => {await notifyChatPublicationWork(tx);+ await notifyChatActionWork(tx, "slack_board_message");await tx.update(chatActions).set({ status: "processed", updatedAt: new Date() }).where(eq(chatActions.id, action.id));});} catch (error) {@@ -28120,17 +28092,9 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {async function processPendingChatMaintenance(limit = 25) {return Promise.allSettled([- processPendingSlackBoardMessages(limit),processPendingGitHubWebhookIngress(limit),processPendingProviderEffects(limit),- slackRegistration.processPendingVerificationMessages(limit),- processPendingReceiptReactions(limit),- processPendingSlackSessionStops(limit),processPendingTelegramMaintenance(limit),- // Slack task starts are an action-backed outbox. Queued work may- // post once; provider-confirmed rows perform Paperclip-only- // admission; a stale in-flight post is quarantined as unknown.- processPendingSlackTaskStarts(limit),processFailedChatRunRetries(limit),]);}@@ -28141,7 +28105,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {// chat_actions as their outbox. Reconcile them before inbound deliveries// so a crashed processing claim is quarantined before the delivery worker// could otherwise replay it.- const actionRecovery = onlyDeliveryId || !maintenance ? null : processPendingChatMaintenance(limit);+ const actionRecovery = onlyDeliveryId || !maintenance ? null : Promise.allSettled([+ processPendingChatMaintenance(limit), processPendingSlackBoardMessages(limit),+ processPendingSlackTaskStarts(limit), processPendingReceiptReactions(limit),+ processPendingSlackSessionStops(limit), slackRegistration.processPendingVerificationMessages(limit),+ ]);const reactionRecovery = processPendingReactionDeliveries(limit,onlyDeliveryId,@@ -30373,6 +30341,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {async (credentialLease) => {await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "slash_task_start");await tx.update(chatActions).set({@@ -30428,6 +30397,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {);}const now = new Date();+ await notifyChatActionWork(tx, "slash_task_start");await tx.update(chatActions).set({@@ -30612,6 +30582,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {: 0) + 1;const [claimed] = await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "slash_task_start");await tx.update(chatActions).set({@@ -30717,6 +30688,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {},);}+ await notifyChatActionWork(tx, "slash_task_start");const rows = await tx.update(chatActions).set({@@ -30778,6 +30750,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {}const [completed] = await db.transaction(async (tx) => {await credentialLease.assertOwned(tx);+ await notifyChatActionWork(tx, "slash_task_start");return tx.update(chatActions).set({@@ -39117,6 +39090,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {processQueuedDeliveries: () => processPendingDeliveries(25, undefined, false),scheduleQueuedPublications: () => processPendingPublications(25, { waitForCompletion: false, maintenance: false }),processPendingChatMaintenance,+ processPendingSlackBoardMessages,+ processPendingSlackTaskStarts,+ processPendingVerificationMessages: slackRegistration.processPendingVerificationMessages,+ nextVerificationMessageAt: slackRegistration.nextVerificationMessageAt,+ nextChatActionAt: (kind: ScheduledChatAction) => nextChatActionAt(db, kind),processPublicationMaintenance,nextInboundDeliveryAt,nextPublicationAt,diff --git a/server/src/services/chat-delivery-work.ts b/server/src/services/chat-delivery-work.tsindex 1872e3b689..ed790894d3 100644--- a/server/src/services/chat-delivery-work.ts+++ b/server/src/services/chat-delivery-work.ts@@ -1,5 +1,5 @@import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";-import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";+import { DELIVERY_QUEUES, type DeliveryQueue } from "./delivery-work-notifications.js";type ChatDeliveryService = {processQueuedDeliveries(): Promise<unknown>;@@ -23,16 +23,7 @@ export function registerChatDeliveryWork({ queue: DELIVERY_QUEUES.chatReceipts, run: service.processPendingSlackFileUploadReceipts, next: service.nextSlackReceiptAt },];const workers = tasks.map(task => {- let next: number | null = null;- const worker = coordinator.register(task.queue, {- retryMs: 1000,- run: () => task.run(),- hasPending: async () => {- next = canRun() ? await task.next() : Date.now() + 1000;- return false;- },- nextRunAt: () => next === null ? null : Math.max(Date.now() + 1000, next),- });+ const worker = registerChatQueueWork(coordinator, task, canRun);if (task.queue === DELIVERY_QUEUES.chatPublications) {// Dispatch returns after reserving endpoint slots. Refill them when a// provider task ends, including failures before its final durable write.@@ -42,3 +33,22 @@ export function registerChatDeliveryWork(});return { ready: Promise.all(workers.map(worker => worker.ready)) };}++/** Empty queues disarm; only known work, uncertain writes, and failures retry. */+export function registerChatQueueWork(+ coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,+ task: { queue: DeliveryQueue; run(): Promise<unknown>; next(): Promise<number | null> },+ canRun: () => boolean,+) {+ let next: number | null = null;+ const worker = coordinator.register(task.queue, {+ retryMs: 1000,+ run: () => task.run(),+ hasPending: async () => {+ next = canRun() ? await task.next() : Date.now() + 1000;+ return false;+ },+ nextRunAt: () => next === null ? null : Math.max(Date.now() + 1000, next),+ });+ return worker;+}diff --git a/server/src/services/chat-receipt-reactions.ts b/server/src/services/chat-receipt-reactions.tsindex c550649fba..3a0487de66 100644--- a/server/src/services/chat-receipt-reactions.ts+++ b/server/src/services/chat-receipt-reactions.ts@@ -1,3 +1,4 @@+import { notifyChatActionWork } from "./chat-work-notifications.js";import { and, eq, inArray, or, sql } from "drizzle-orm";import {chatActions,@@ -205,6 +206,7 @@ export async function stageReceiptReactionRemovals(];});if (removals.length === 0) return [];+ await notifyChatActionWork(tx, "receipt_reaction");const removalDeliveryIds = new Set(removals.map((removal) => removal.deliveryId),);diff --git a/server/src/services/chat-slack-registration.ts b/server/src/services/chat-slack-registration.tsindex a42909e7ae..905c7ab034 100644--- a/server/src/services/chat-slack-registration.ts+++ b/server/src/services/chat-slack-registration.ts@@ -1,3 +1,4 @@+import { notifyChatVerificationWork } from "./chat-work-notifications.js";import { createAgentAvatarPool } from "./agent-avatar-pool.js";import type { AgentAvatarRequest } from "./agent-avatars.js";import { removeSlackRegistration } from "./chat-slack-registration-cleanup.js";@@ -111,6 +112,7 @@ export function slackChatRegistrationService(db: Db, options: {...(ids[key] ? { existingRef: { secretId: ids[key], configPath: `slack_registration.${key}` } } : {}) });ids[key] = result.secret.id;}+ await notifyChatVerificationWork(tx);await tx.update(chatSlackRegistrations).set({ ...patch, secretIds: ids, updatedAt: new Date() }).where(eq(chatSlackRegistrations.endpointId, row.endpointId));if (account) {const saved = (await tx.select().from(chatEndpoints).where(and(eq(chatEndpoints.id, row.endpointId), eq(chatEndpoints.companyId, row.companyId))).for("update"))[0];@@ -334,6 +336,7 @@ export function slackChatRegistrationService(db: Db, options: {const saved = (await tx.select().from(chatEndpoints).where(and(eq(chatEndpoints.id, row.endpointId), eq(chatEndpoints.companyId, row.companyId))).for("update"))[0];const registrationNow = (await tx.select().from(chatSlackRegistrations).where(eq(chatSlackRegistrations.endpointId, row.endpointId)))[0];if (!saved || saved.status === "archived" || registrationNow?.status === "removed" || registrationNow?.requestId !== row.requestId) throw conflict("Slack setup changed");+ await notifyChatVerificationWork(tx);await tx.update(chatEndpoints).set({ setup: { ...saved.setup, slackAccount: account }, updatedAt: new Date() }).where(eq(chatEndpoints.id, row.endpointId));await lease.assertOwned(tx);});@@ -397,12 +400,26 @@ export function slackChatRegistrationService(db: Db, options: {await sendSetupMessage(row, { userId: account.paperclipUserId, sessionId: null, bypassPermissionCheck: false }, lease, "verification");});}- async function processPendingVerificationMessages(limit = 25) {- const pending = await db.select({ endpointId: chatEndpoints.id }).from(chatEndpoints)+ function pendingVerificationMessage() {+ // A pending verification cannot progress until linking, welcome, and an+ // authenticated callback finish. Their commits wake this lane.+ return and(eq(chatSlackRegistrations.status, "configured"),+ inArray(chatEndpoints.status, ["verifying", "active"]),+ sql`${chatEndpoints.setup}->'slackAccount'->>'status' = 'linked'`,+ sql`${chatEndpoints.setup}->>'webhookVerifiedAt' is not null`,+ sql`coalesce(${chatEndpoints.setup}->'slackAccount'->>'welcomeStatus', '') not in ('pending', 'sending')`,+ sql`${chatEndpoints.setup}->'slackAccount'->>'verificationStatus' in ('pending', 'sending')`);+ }+ function pendingVerificationMessages(limit: number) {+ return db.select({ endpointId: chatEndpoints.id }).from(chatEndpoints).innerJoin(chatSlackRegistrations, and(eq(chatSlackRegistrations.endpointId, chatEndpoints.id), eq(chatSlackRegistrations.companyId, chatEndpoints.companyId)))- .where(and(eq(chatSlackRegistrations.status, "configured"), inArray(chatEndpoints.status, ["verifying", "active"]),- sql`${chatEndpoints.setup}->'slackAccount'->>'verificationStatus' in ('pending', 'sending')`)).limit(limit);- for (const row of pending) {+ .where(pendingVerificationMessage()).limit(limit);+ }+ async function nextVerificationMessageAt(): Promise<number | null> {+ return (await pendingVerificationMessages(1)).length ? Date.now() : null;+ }+ async function processPendingVerificationMessages(limit = 25) {+ for (const row of await pendingVerificationMessages(limit)) {try { await notifyVerified(row.endpointId); }catch { logger.warn({ endpointId: row.endpointId }, "Slack verification message could not be processed"); }}@@ -437,7 +454,10 @@ export function slackChatRegistrationService(db: Db, options: {return;}await lease.assertOwned();- await db.update(chatSlackRegistrations).set({ status: "configured", errorCode: null, updatedAt: new Date() }).where(eq(chatSlackRegistrations.endpointId, endpointId));+ await db.transaction(async tx => {+ await notifyChatVerificationWork(tx);+ await tx.update(chatSlackRegistrations).set({ status: "configured", errorCode: null, updatedAt: new Date() }).where(eq(chatSlackRegistrations.endpointId, endpointId));+ });// Runtime now owns a separate vaulted copy. Keep the registration IDs until// deletion succeeds so cleanup itself is retryable.await cleanupStaged(endpointId, lease);@@ -505,7 +525,7 @@ export function slackChatRegistrationService(db: Db, options: {if (!row) throw notFound("Slack connection not found");return `/${encodeURIComponent(row.prefix)}/apps/chat/connect?provider=slack&resume=${encodeURIComponent(endpointId)}`;}- return { create, install, pending, expiredReturn, complete, cleanup, returnPath, registration, notifyVerified, processPendingVerificationMessages,+ return { create, install, pending, expiredReturn, complete, cleanup, returnPath, registration, notifyVerified, processPendingVerificationMessages, nextVerificationMessageAt,close: async () => { await avatarPool?.close(); },resume: (endpointId: string, actor: SlackSetupActor) => options.withLock(endpointId, lease => resumeLocked(endpointId, actor, lease)) };}diff --git a/server/src/services/chat-work-notifications.ts b/server/src/services/chat-work-notifications.tsindex f6caa18e0b..50da6074d5 100644--- a/server/src/services/chat-work-notifications.ts+++ b/server/src/services/chat-work-notifications.ts@@ -10,7 +10,25 @@ export async function notifyChatPublicationWork(tx: object): Promise<void> {await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatReceipts);}+export const CHAT_ACTION_QUEUES = {+ slack_board_message: DELIVERY_QUEUES.chatBoardMessages,+ slash_task_start: DELIVERY_QUEUES.chatTaskStarts,+ receipt_reaction: DELIVERY_QUEUES.chatReceiptReactions,+ slack_session_stop: DELIVERY_QUEUES.chatSessionStops,+} as const;+export type ScheduledChatAction = keyof typeof CHAT_ACTION_QUEUES;++export async function notifyChatActionWork(tx: object, kind: ScheduledChatAction): Promise<void> {+ await notifyDeliveryWork(tx, CHAT_ACTION_QUEUES[kind]);+}++export async function notifyChatVerificationWork(tx: object): Promise<void> {+ await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatVerification);+}+export async function notifyChatEndpointWork(tx: object): Promise<void> {await notifyChatDeliveryWork(tx);await notifyChatPublicationWork(tx);+ for (const queue of Object.values(CHAT_ACTION_QUEUES)) await notifyDeliveryWork(tx, queue);+ await notifyChatVerificationWork(tx);}diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.tsindex 98dfb361d4..686d860f02 100644--- a/server/src/services/delivery-work-notifications.ts+++ b/server/src/services/delivery-work-notifications.ts@@ -2,6 +2,11 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";/** Topics for existing durable queues; register intent before their writes. */export const DELIVERY_QUEUES = {+ chatBoardMessages: "chat-board-messages",+ chatTaskStarts: "chat-task-starts",+ chatReceiptReactions: "chat-receipt-reactions",+ chatSessionStops: "chat-session-stops",+ chatVerification: "chat-verification",chatInbound: "chat-inbound",chatPublications: "chat-publications",chatReceipts: "chat-slack-receipts",diff --git a/server/src/services/slack-board-messages.ts b/server/src/services/slack-board-messages.tsindex b8fc4ddd8f..a00a8f0f75 100644--- a/server/src/services/slack-board-messages.ts+++ b/server/src/services/slack-board-messages.ts@@ -1,4 +1,4 @@-import { notifyChatPublicationWork } from "./chat-work-notifications.js";+import { notifyChatPublicationWork, notifyChatActionWork } from "./chat-work-notifications.js";import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm";import {assets,@@ -102,6 +102,7 @@ export async function mirrorSlackBoardComment(if (!principal) {throw forbidden("Link your Slack account to this connection before sending a message to its Slack thread");}+ if (options.wakeAgent) await notifyChatActionWork(db, "slack_board_message");const [receipt] = await db.insert(chatActions).values({