Multiplayer agent harness for work.
Verify Slack state before repeating cut deliveries and sweep silent terminal runs (#2053)
Regan Bell93c64f516b88
diff --git a/src/delivery/delivery-store.ts b/src/delivery/delivery-store.tsindex 39d22f4f3..a81afbe74 100644--- a/src/delivery/delivery-store.ts+++ b/src/delivery/delivery-store.ts@@ -50,6 +50,7 @@ export interface DeliveryStore {ackByKey(idempotencyKey: string, at: number): Promise<void>;setEditRefByKey(idempotencyKey: string, editRef: string): Promise<void>;get(id: string): Promise<Delivery | null>;+ existingKeys(idempotencyKeys: string[]): Promise<Set<string>>;recordRecipientThread(id: string, recipientThreadRef: string, at: number): Promise<void>;listByRecipientThread(recipientThreadRef: string, opts?: { limit?: number }): Promise<Delivery[]>;listBySourceSession(sourceSessionId: string, sourceThreadRef: string, opts?: { limit?: number }): Promise<Delivery[]>;@@ -176,6 +177,9 @@ export function createDeliveryStore(opts?: { maxAgeMs?: number }): DeliveryStoreasync get(id) {return deliveries.get(id) ?? null;},+ async existingKeys(idempotencyKeys) {+ return new Set(idempotencyKeys.filter((key) => byKey.has(key)));+ },async recordRecipientThread(id, recipientThreadRef, at) {const d = deliveries.get(id);if (!d || d.destination.type !== "principal") return;diff --git a/src/delivery/postgres-delivery-store.ts b/src/delivery/postgres-delivery-store.tsindex 5fa46ffbf..daef5355a 100644--- a/src/delivery/postgres-delivery-store.ts+++ b/src/delivery/postgres-delivery-store.ts@@ -218,6 +218,13 @@ export function createPostgresDeliveryStore(connectionString: string, opts?: { mconst rows = await q("SELECT * FROM deliveries WHERE id = $1", [id]);return rows[0] ? rowToDelivery(rows[0]) : null;},+ async existingKeys(idempotencyKeys) {+ if (!idempotencyKeys.length) return new Set();+ const rows = await q("SELECT idempotency_key FROM deliveries WHERE idempotency_key = ANY($1::text[])", [+ idempotencyKeys,+ ]);+ return new Set(rows.map((r) => r.idempotency_key as string));+ },async recordRecipientThread(id, recipientThreadRef, at) {await query(`UPDATE deliveriesdiff --git a/src/delivery/run-result-delivery.ts b/src/delivery/run-result-delivery.tsindex 171989a0b..ec060fcd0 100644--- a/src/delivery/run-result-delivery.ts+++ b/src/delivery/run-result-delivery.ts@@ -138,24 +138,53 @@ export async function recordRunFailureEntry(sessions: TurnFailureSessions, run:}}+const RUN_RESULT_SWEEP_LOOKBACK_MS = 60 * 60_000;+const RUN_RESULT_SWEEP_GRACE_MS = 60_000;+const RUN_RESULT_SWEEP_PAGE = 200;++export interface RunResultRecovery {+ sweep(now?: number): Promise<number>;+}+export function wireRunResultDeliveries(- runs: RunStore,- deliveries: DeliveryStore,+ runs: Pick<RunStore, "onTerminal" | "terminalFinished">,+ deliveries: Pick<DeliveryStore, "enqueue" | "existingKeys">,tasks?: TaskStore,adminUrlFor?: AdminUrlFor,sessions?: TurnFailureSessions,-): void {- runs.onTerminal((run) => {+): RunResultRecovery {+ const recover = async (run: Run): Promise<boolean> => {if (sessions) {void recordRunFailureEntry(sessions, run).catch(reportFailureAs("delivery: record turn_failure entry", undefined, `run=${run.id}`),);}- void (async () => {- const taskList = tasks ? await tasks.list({ originRunId: run.id }) : [];- const delivery = runResultDelivery(run, taskList, adminUrlFor);- if (!delivery) return;- await deliveries.enqueue(delivery);- })().catch(reportFailureAs("delivery: enqueue recovery delivery", undefined, `run=${run.id}`));+ const taskList = tasks ? await tasks.list({ originRunId: run.id }) : [];+ const delivery = runResultDelivery(run, taskList, adminUrlFor);+ if (!delivery) return false;+ await deliveries.enqueue(delivery);+ return true;+ };+ runs.onTerminal((run) => {+ void recover(run).catch(reportFailureAs("delivery: enqueue recovery delivery", undefined, `run=${run.id}`));});+ return {+ async sweep(now = Date.now()) {+ const before = now - RUN_RESULT_SWEEP_GRACE_MS;+ let cursor = { finishedAt: now - RUN_RESULT_SWEEP_LOOKBACK_MS, id: "" };+ let recovered = 0;+ for (;;) {+ const batch = await runs.terminalFinished(cursor, before, RUN_RESULT_SWEEP_PAGE);+ const existing = await deliveries.existingKeys(batch.map((run) => `run:${run.id}`));+ for (const run of batch) {+ if (existing.has(`run:${run.id}`) || !(await recover(run))) continue;+ console.error(`[delivery] terminal run ${run.id} had no outbox row — enqueued its recovery delivery`);+ recovered += 1;+ }+ const last = batch.at(-1);+ if (!last || batch.length < RUN_RESULT_SWEEP_PAGE) return recovered;+ cursor = { finishedAt: last.finishedAt!, id: last.id };+ }+ },+ };}diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.tsindex 88ff6eb43..c49b78926 100644--- a/src/runs/memory-run-store.ts+++ b/src/runs/memory-run-store.ts@@ -186,6 +186,18 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti.sort((a, b) => a.id.localeCompare(b.id)).slice(0, limit);},+ async terminalFinished(after, beforeMs, limit) {+ return [...runs.values()]+ .filter(+ (run) =>+ isTerminal(run.status) &&+ run.finishedAt !== null &&+ run.finishedAt <= beforeMs &&+ (run.finishedAt > after.finishedAt || (run.finishedAt === after.finishedAt && run.id > after.id)),+ )+ .sort((a, b) => a.finishedAt! - b.finishedAt! || (a.id < b.id ? -1 : 1))+ .slice(0, limit);+ },async markReturned(runId) {returned.add(runId);},diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.tsindex 4273fa53e..4824f9ab0 100644--- a/src/runs/postgres-run-store.ts+++ b/src/runs/postgres-run-store.ts@@ -126,6 +126,12 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_created ON runs(created_at DESC)`,],},+ {+ id: "runs/store/0006-terminal-finished",+ statements: [+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_terminal_finished ON runs(finished_at, id) WHERE status IN ('done','failed')`,+ ],+ },],[{@@ -401,6 +407,15 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla);return rows.map(rowToRun);},+ async terminalFinished(after, beforeMs, limit) {+ const { rows } = await q(+ `SELECT * FROM runs WHERE status IN ('done','failed') AND finished_at <= $1+ AND (finished_at > $2 OR (finished_at = $2 AND id > $3))+ ORDER BY finished_at, id LIMIT $4`,+ [beforeMs, after.finishedAt, after.id, limit],+ );+ return rows.map(rowToRun);+ },async markReturned(runId) {await q("UPDATE runs SET returned_at = $2 WHERE id = $1 AND status IN ('done','failed')", [runId, Date.now()]);},diff --git a/src/runs/run-store.ts b/src/runs/run-store.tsindex 347c1425a..fcc328196 100644--- a/src/runs/run-store.ts+++ b/src/runs/run-store.ts@@ -39,6 +39,11 @@ export interface Run {finishedAt: number | null;}+interface TerminalCursor {+ finishedAt: number;+ id: string;+}+export interface EnqueueInput {sessionId: string;request: OrchestratorInput;@@ -84,6 +89,7 @@ export interface RunStore {latestForThread(threadRef: string, opts?: { excludePrivateMessages?: boolean }): Promise<Run | null>;pendingReturns(limit?: number, afterId?: string): Promise<Run[]>;+ terminalFinished(after: TerminalCursor, beforeMs: number, limit: number): Promise<Run[]>;markReturned(runId: string): Promise<void>;deferReturn(runId: string, delayMs: number): Promise<void>;diff --git a/src/slack/deliveries.ts b/src/slack/deliveries.tsindex deeeafba4..d3891b57d 100644--- a/src/slack/deliveries.ts+++ b/src/slack/deliveries.ts@@ -11,10 +11,15 @@ import {createDeliveryTracker,createThreadTracker,deliverWithRetry,+ deliveryMetadata,dmThreadRef,openConversationFor,findPostedByKey,+ messageExists,+ messagePinned,parseDeliveryTarget,+ reactionPresent,+ statusPlaceholderKey,postWithVerify,recoveryVerifyOldest,renderTaskList,@@ -47,6 +52,7 @@ const PERMANENT_POST_ERRORS = new Set([const DELIVERY_CLAIM_MARGIN_MS = 2_000;const RUN_RECOVERY_GRACE_MS = 15_000;+const STATUS_PLACEHOLDER_LOOKBACK_MS = 6 * 3_600_000;function mergeSlackApiMs(body: unknown, slackApiMs: number | undefined): unknown {if (slackApiMs === undefined) return body;@@ -236,10 +242,11 @@ export function createDeliveryPoller(deps: {await deliverKeychainCard(core, client, d, channel, threadTs, deps.webUiPublicUrl);return undefined;}+ const recovered = recoveredRow(d);if (d.destination.react) {- const { failed } = await applyReactions(client, channel, d.destination.react.messageTs, [- d.destination.react.emoji,- ]);+ const { messageTs, emoji } = d.destination.react;+ if (recovered && (await reactionPresent(client, channel, messageTs, emoji))) return undefined;+ const { failed } = await applyReactions(client, channel, messageTs, [emoji]);if (failed.length)console.error(`[slack-plugin] delivery ${d.id} reaction(s) failed: ${failed.join(", ")} (check reactions:write / message ts)`,@@ -248,6 +255,7 @@ export function createDeliveryPoller(deps: {}if (d.destination.pin) {const { messageTs, remove } = d.destination.pin;+ if (recovered && (await messagePinned(client, channel, messageTs)) === !remove) return undefined;try {if (remove) await client.pins.remove({ channel, timestamp: messageTs });else await client.pins.add({ channel, timestamp: messageTs });@@ -261,8 +269,10 @@ export function createDeliveryPoller(deps: {return undefined;}if (d.destination.delete) {+ const { messageTs } = d.destination.delete;+ if (recovered && (await messageExists(client, channel, messageTs)) === false) return undefined;try {- await client.chat.delete({ channel, ts: d.destination.delete.messageTs });+ await client.chat.delete({ channel, ts: messageTs });} catch (err) {console.error(`[slack-plugin] delivery ${d.id} delete failed: ${slackErrorCode(err) ?? (err as Error).message} (own messages only)`,@@ -270,6 +280,19 @@ export function createDeliveryPoller(deps: {}return undefined;}+ let editRef = d.destination.editRef;+ if (!editRef && runId) {+ const placeholder = await findPostedByKey(+ client,+ { channel, ...(threadTs ? { thread_ts: threadTs } : {}) },+ statusPlaceholderKey(runId),+ String((d.createdAt - STATUS_PLACEHOLDER_LOOKBACK_MS) / 1000),+ ).catch(swallowAs("slack: status-placeholder probe", undefined));+ if (placeholder) {+ await core.reportRunEditRef(runId, placeholder.ts);+ editRef = placeholder.ts;+ }+ }let text = toSlackMrkdwn(runId ? cleanAgentReplyForSlack(d.text).text : stripSlackDirectives(d.text));const replayAttachments = async (root?: string): Promise<void> => {if (!d.attachments?.length) return;@@ -304,11 +327,11 @@ export function createDeliveryPoller(deps: {if (!text.trim() && !(messageFooter.length && d.attachments?.length)) {if (taskList) {let preserved = false;- if (d.destination.editRef) {+ if (editRef) {try {await client.chat.update({channel,- ts: d.destination.editRef,+ ts: editRef,text: taskList,blocks: [{ type: "section", text: { type: "mrkdwn", text: taskList } }],...botIdentityArgs(),@@ -324,9 +347,9 @@ export function createDeliveryPoller(deps: {blocks: [{ type: "section", text: { type: "mrkdwn", text: taskList } }],});}- } else if (d.destination.editRef) {+ } else if (editRef) {await client.chat- .delete({ channel, ts: d.destination.editRef })+ .delete({ channel, ts: editRef }).catch(swallowAs("slack: delete status placeholder", undefined));}await replayAttachments(threadTs);@@ -335,7 +358,7 @@ export function createDeliveryPoller(deps: {}const verifyOldest = recoveryVerifyOldest(typeof d.createdAt === "number" ? d.createdAt : undefined,- d.destination.editRef,+ editRef,);const deliveredMarker = (): Promise<{ ts: string; channel: string } | undefined> =>findPostedByKey(@@ -344,16 +367,17 @@ export function createDeliveryPoller(deps: {d.idempotencyKey ?? d.id,verifyOldest ?? String(Date.now() / 1000 - 60),).catch(swallowAs("slack: delivered-marker probe", undefined));- if (d.destination.editRef) {+ if (editRef) {const unfurlLinks = runId ? false : d.destination.unfurlLinks;try {const alreadyDelivered = d.attachments?.length ? await deliveredMarker() : undefined;await client.chat.update({channel,- ts: d.destination.editRef,+ ts: editRef,text,...(footerBlocks ? { blocks: footerBlocks } : {}),...botIdentityArgs(),+ metadata: deliveryMetadata(d.idempotencyKey ?? d.id),...(unfurlLinks !== undefined ? { unfurl_links: unfurlLinks, unfurl_media: unfurlLinks } : {}),});if (threadTs) threads.mark(channel, threadTs, true);diff --git a/src/slack/delivery.ts b/src/slack/delivery.tsindex 20fb7d7c8..7263a47d3 100644--- a/src/slack/delivery.ts+++ b/src/slack/delivery.ts@@ -470,6 +470,74 @@ export async function findPostedByKey(return undefined;}+export function statusPlaceholderKey(runId: string): string {+ return `status:run:${runId}`;+}++interface SlackStateClient {+ reactions: { get(args: { channel: string; timestamp: string; full: boolean }): Promise<unknown> };+ pins: { list(args: { channel: string }): Promise<unknown> };+ conversations: {+ replies(args: {+ channel: string;+ ts: string;+ oldest: string;+ latest: string;+ inclusive: boolean;+ limit: number;+ }): Promise<unknown>;+ };+}++export async function reactionPresent(+ client: Pick<SlackStateClient, "reactions">,+ channel: string,+ timestamp: string,+ name: string,+): Promise<boolean | undefined> {+ try {+ const res = (await client.reactions.get({ channel, timestamp, full: true })) as {+ message?: { reactions?: Array<{ name?: string; count?: number }> };+ };+ return (res.message?.reactions ?? []).some((r) => r.name === name && (r.count ?? 0) > 0);+ } catch {+ return undefined;+ }+}++export async function messagePinned(+ client: Pick<SlackStateClient, "pins">,+ channel: string,+ timestamp: string,+): Promise<boolean | undefined> {+ try {+ const res = (await client.pins.list({ channel })) as { items?: Array<{ message?: { ts?: string } }> };+ return (res.items ?? []).some((item) => item.message?.ts === timestamp);+ } catch {+ return undefined;+ }+}++export async function messageExists(+ client: Pick<SlackStateClient, "conversations">,+ channel: string,+ timestamp: string,+): Promise<boolean | undefined> {+ try {+ const res = (await client.conversations.replies({+ channel,+ ts: timestamp,+ oldest: timestamp,+ latest: timestamp,+ inclusive: true,+ limit: 1,+ })) as { messages?: Array<{ ts?: string }> };+ return (res.messages ?? []).some((m) => m.ts === timestamp);+ } catch (err) {+ return (err as { data?: { error?: string } }).data?.error === "thread_not_found" ? false : undefined;+ }+}+export function recoveryVerifyOldest(createdAt: number | undefined, editRef: string | undefined): string | undefined {const bounds: number[] = [];if (typeof createdAt === "number") bounds.push(createdAt / 1000 - 60);diff --git a/src/slack/lib.ts b/src/slack/lib.tsindex fb294bed4..815814be6 100644--- a/src/slack/lib.ts+++ b/src/slack/lib.ts@@ -140,6 +140,10 @@ export {postWithVerify,findPostedByKey,recoveryVerifyOldest,+ statusPlaceholderKey,+ reactionPresent,+ messagePinned,+ messageExists,deliveryMetadata,type DeliveryMetadata,type PostedPart,diff --git a/src/slack/turn-handler.ts b/src/slack/turn-handler.tsindex cad339348..68c24058a 100644--- a/src/slack/turn-handler.ts+++ b/src/slack/turn-handler.ts@@ -34,6 +34,7 @@ import {dedupedRun,deliveryCandidatesFor,deliveryMetadata,+ statusPlaceholderKey,channelThreadRef,dmThreadRef,downloadSlackFile,@@ -395,13 +396,14 @@ export function createTurnHandler(deps: {}let ack: AckPresenter | undefined;+ const statusKey = (): string | undefined => (queuedRunId ? statusPlaceholderKey(queuedRunId) : undefined);const startAck = (): AckPresenter =>(ack ??= createAckPresenter({taskManaged,postAck: async (text) => {const rendered = toSlackMrkdwn(text);if (await taskList?.addLead(rendered)) return;- const ts = await postReply(rendered);+ const ts = await postReply(rendered, undefined, statusKey());if (ts) {await taskList?.attach(ts, rendered);}@@ -424,7 +426,7 @@ export function createTurnHandler(deps: {if (!inc.unprompted) startAck();if (!inc.unprompted) {taskList = createTaskListPresenter({- post: (text, blocks) => postReply(text, blocks),+ post: (text, blocks) => postReply(text, blocks, statusKey()),update: (ts, text, blocks, metadata) =>client.chat.update({diff --git a/src/wiring.ts b/src/wiring.tsindex 424dc2b98..8fb402aba 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -2001,7 +2001,7 @@ export function buildApp(? (sessionId: string): string | undefined =>uuidId.test(sessionId) ? adminSessionUrl(recoveryAdminBase, sessionId) : undefined: undefined;- wireRunResultDeliveries(runs, deliveries, tasks, recoveryAdminUrlFor, sessions);+ const runResultRecovery = wireRunResultDeliveries(runs, deliveries, tasks, recoveryAdminUrlFor, sessions);const idempotency = createIdempotencyStore(artifactMap<IdempotencyRecord>("idempotency"));const skillFetcher = createGitFetcher(keychain@@ -2313,6 +2313,14 @@ export function buildApp(1_000,{ label: "session-returns", immediate: true },);+ const runResultSweeper = createSweeper(+ () =>+ advisoryLock.tryWithLock+ ? advisoryLock.tryWithLock("run-result-sweep", () => runResultRecovery.sweep())+ : advisoryLock.withLock("run-result-sweep", () => runResultRecovery.sweep()),+ 60_000,+ { label: "run-results" },+ );const orphanedSignalSweeper = createSweeper(async () => {for (const runId of await runSignals.pendingRunIds()) {@@ -2669,6 +2677,7 @@ export function buildApp(swarms?.start();orphanedSignalSweeper.start();sessionReturnSweeper.start();+ runResultSweeper.start();approvalDeliverySweeper.start();};if (backgroundStopping)@@ -2698,6 +2707,7 @@ export function buildApp(swarms?.stop(),orphanedSignalSweeper.stop(),sessionReturnSweeper.stop(),+ runResultSweeper.stop(),approvalDeliverySweeper.stop(),...workers.map((worker) => worker.stopClaims()),];diff --git a/test/delivery-store-contract.ts b/test/delivery-store-contract.tsindex 687d4d146..c46b3a764 100644--- a/test/delivery-store-contract.ts+++ b/test/delivery-store-contract.ts@@ -235,6 +235,13 @@ export async function exerciseDeliveryStore(store: DeliveryStore): Promise<void>"an expired claim re-surfaces (at-least-once)",);await store.ack(abandoned.id, 700);++ assert.deepEqual(+ [...(await store.existingKeys(["fire-1", "run:r-2", "fire-race-2", "never-enqueued"]))].sort(),+ ["fire-1", "fire-race-2", "run:r-2"],+ "existing keys cover enqueued rows and ack tombstones alike",+ );+ assert.deepEqual(await store.existingKeys([]), new Set());}export async function exerciseDeliveryExpiry(makeStore: (opts: { maxAgeMs: number }) => DeliveryStore): Promise<void> {diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.tsindex a5d2c2911..9b4bc0d1b 100644--- a/test/postgres-store.test.ts+++ b/test/postgres-store.test.ts@@ -1797,6 +1797,40 @@ test("pg run store: Unicode stays jsonb-safe through enqueue, edit and both stee}});+test("pg run store: terminalFinished lists finished runs by (finishedAt, id) up to the bound", { skip }, async () => {+ const { runs, close } = createPostgresRunStore(URL!);+ const tag = randomUUID();+ try {+ const ids: string[] = [];+ for (const ref of ["a", "b", "c"]) {+ const thread = `${tag}-${ref}`;+ const run = (await runs.enqueue({ sessionId: thread, request: turn(ref) })).run;+ const claimed = await runs.claimForSession(thread, "w", 5_000);+ await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply: ref });+ ids.push(run.id);+ }+ const open = (await runs.enqueue({ sessionId: `${tag}-open`, request: turn("open") })).run;+ await runs.claimForSession(`${tag}-open`, "w", 5_000);+ const far = Date.now() + 60_000;+ const started = (await runs.get(ids[0]!))!.startedAt! - 1;+ const listed = await runs.terminalFinished({ finishedAt: started, id: "" }, far, 10_000);+ const mine = listed.filter((run) => run.sessionId.startsWith(tag));+ assert.deepEqual(new Set(mine.map((run) => run.id)), new Set(ids));+ assert.ok(!listed.some((run) => run.id === open.id));+ for (let i = 1; i < listed.length; i++) {+ const prev = listed[i - 1]!;+ const next = listed[i]!;+ assert.ok(prev.finishedAt! < next.finishedAt! || (prev.finishedAt === next.finishedAt && prev.id < next.id));+ }+ const first = mine[0]!;+ const after = await runs.terminalFinished({ finishedAt: first.finishedAt!, id: first.id }, far, 10_000);+ assert.ok(!after.some((run) => run.id === first.id), "the cursor row itself is excluded");+ assert.deepEqual(await runs.terminalFinished({ finishedAt: started, id: "" }, first.finishedAt! - 1, 10_000), []);+ } finally {+ await close();+ }+});+test("pg run store: a session_busy completion frees the dedup key so the same key runs again", { skip }, async () => {const { runs, close } = createPostgresRunStore(URL!);try {diff --git a/test/run-result-delivery.test.ts b/test/run-result-delivery.test.tsindex 133e7ce9a..ca1534bd4 100644--- a/test/run-result-delivery.test.ts+++ b/test/run-result-delivery.test.ts@@ -255,6 +255,80 @@ test("wired stores: a completed turn lands in the outbox unless the live path ac);});+test("sweep enqueues the recovery delivery for a terminal run whose outbox row never landed", async () => {+ const { runs } = createMemoryRunStore();+ const deliveries = createDeliveryStore();+ const bypassed = (await runs.enqueue({ sessionId: "sA", request: turn("a", "C9:171.001") })).run;+ const claimed = await runs.claim("w1", 5_000);+ await runs.complete(bypassed.id, claimed?.leaseToken ?? "", { status: "ok", reply: "lost on the way out" });+ const recovery = wireRunResultDeliveries(runs, deliveries);+ assert.deepEqual(await deliveries.pending("slack"), [], "wired after the fact: nothing enqueued");++ assert.equal(await recovery.sweep(Date.now() + 120_000), 1);+ const pending = await deliveries.pending("slack");+ assert.deepEqual(+ pending.map((d) => [d.idempotencyKey, d.text]),+ [[`run:${bypassed.id}`, "lost on the way out"]],+ );+ assert.equal(await recovery.sweep(Date.now() + 120_000), 0, "a second sweep finds the row and leaves it alone");+});++test("sweep leaves runs alone when their delivery already exists, was acked live, or is still inside the grace window", async () => {+ const { runs } = createMemoryRunStore();+ const deliveries = createDeliveryStore();+ const recovery = wireRunResultDeliveries(runs, deliveries);+ const finish = async (threadRef: string, reply: string): Promise<Run> => {+ const run = (await runs.enqueue({ sessionId: threadRef, request: turn(reply, "C9:171.001") })).run;+ const claimed = await runs.claimForSession(threadRef, "w1", 5_000);+ await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply });+ return run;+ };+ const enqueued = await finish("sA", "already enqueued");+ const live = await finish("sB", "delivered live");+ await deliveries.ackByKey(`run:${live.id}`, Date.now());+ const silent = (await runs.enqueue({ sessionId: "sC", request: turn("c", "C9:171.001") })).run;+ await runs.complete(silent.id, (await runs.claimForSession("sC", "w1", 5_000))?.leaseToken ?? "", {+ status: "silent",+ });+ await new Promise((r) => setTimeout(r, 0));+ assert.deepEqual(+ (await deliveries.pending("slack")).map((d) => d.idempotencyKey),+ [`run:${enqueued.id}`],+ );++ assert.equal(await recovery.sweep(Date.now() + 120_000), 0);+ assert.deepEqual(+ (await deliveries.pending("slack")).map((d) => d.idempotencyKey),+ [`run:${enqueued.id}`],+ );++ const fresh = await finish("sD", "just finished");+ await deliveries.ackByKey(`run:${fresh.id}`, Date.now());+ const bare = createDeliveryStore();+ const bareRecovery = wireRunResultDeliveries(runs, bare);+ assert.equal(await bareRecovery.sweep(Date.now()), 0, "runs finished inside the grace window are not swept yet");+ assert.equal(await bareRecovery.sweep(Date.now() + 120_000), 3, "the same runs sweep once the grace window passes");+});++test("sweep pages through terminal runs by (finishedAt, id) without skipping ties", async () => {+ const { runs } = createMemoryRunStore();+ const deliveries = createDeliveryStore();+ const recovery = wireRunResultDeliveries(runs, deliveries);+ const finished = new Set<string>();+ for (let i = 0; i < 205; i++) {+ const run = (await runs.enqueue({ sessionId: `s${i}`, request: turn(`r${i}`, "C9:171.001") })).run;+ const claimed = await runs.claimForSession(`s${i}`, "w1", 5_000);+ await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply: `r${i}` });+ finished.add(`run:${run.id}`);+ }+ await new Promise((r) => setTimeout(r, 0));+ const swept = createDeliveryStore();+ const sweptRecovery = wireRunResultDeliveries(runs, swept);+ assert.equal(await sweptRecovery.sweep(Date.now() + 120_000), 205);+ assert.deepEqual(new Set((await swept.pending("slack")).map((d) => d.idempotencyKey)), finished);+ assert.equal(await recovery.sweep(Date.now() + 120_000), 0);+});+test("wired stores: a parked run lands a durable, non-ackable failure note", async () => {const { runs } = createMemoryRunStore();const deliveries = createDeliveryStore();diff --git a/test/run-store.test.ts b/test/run-store.test.tsindex 81a0f5236..53abb9ac0 100644--- a/test/run-store.test.ts+++ b/test/run-store.test.ts@@ -35,6 +35,37 @@ for (const backend of backends) {]);});+ test(`[${backend.name}] terminalFinished pages finished runs by (finishedAt, id) within the bound`, async () => {+ const { runs } = backend.make();+ const finished: string[] = [];+ for (const ref of ["a", "b", "c"]) {+ const run = (await runs.enqueue({ sessionId: ref, request: turn(ref) })).run;+ const claimed = await runs.claimForSession(ref, "w", 5_000);+ await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply: ref });+ finished.push(run.id);+ }+ await runs.enqueue({ sessionId: "pending", request: turn("pending") });+ const running = (await runs.enqueue({ sessionId: "running", request: turn("running") })).run;+ await runs.claimForSession("running", "w", 5_000);+ const far = Date.now() + 60_000;+ const all = await runs.terminalFinished({ finishedAt: 0, id: "" }, far, 10);+ assert.deepEqual(new Set(all.map((run) => run.id)), new Set(finished), "only terminal runs are listed");+ assert.ok(!all.some((run) => run.id === running.id));+ for (let i = 1; i < all.length; i++) {+ const prev = all[i - 1]!;+ const next = all[i]!;+ assert.ok(prev.finishedAt! < next.finishedAt! || (prev.finishedAt === next.finishedAt && prev.id < next.id));+ }+ const first = await runs.terminalFinished({ finishedAt: 0, id: "" }, far, 2);+ const rest = await runs.terminalFinished({ finishedAt: first[1]!.finishedAt!, id: first[1]!.id }, far, 2);+ assert.deepEqual(+ [...first, ...rest].map((run) => run.id),+ all.map((run) => run.id),+ "keyset paging is gapless",+ );+ assert.deepEqual(await runs.terminalFinished({ finishedAt: 0, id: "" }, all[0]!.finishedAt! - 1, 10), []);+ });+test(`[${backend.name}] enqueue dedups by dedup key`, async () => {const { runs } = backend.make();const a = await runs.enqueue({ sessionId: "s1", request: turn("hi"), dedupKey: "k1" });diff --git a/test/slack-delivery-recovery.test.ts b/test/slack-delivery-recovery.test.tsnew file mode 100644index 000000000..91c26d01b--- /dev/null+++ b/test/slack-delivery-recovery.test.ts@@ -0,0 +1,204 @@+import assert from "node:assert/strict";+import { test } from "node:test";+import { createDeliveryPoller } from "../src/slack/deliveries.ts";+import { statusPlaceholderKey } from "../src/slack/lib.ts";++const RECOVERED_AGE_MS = 60_000;++function harness(opts: {+ thread?: Array<Record<string, unknown>>;+ reactions?: Array<{ name: string; count: number }>;+ pinned?: string[];+ repliesError?: string;+}) {+ const calls: Array<{ method: string; args: Record<string, unknown> }> = [];+ const record =+ (method: string, result: unknown = {}) =>+ async (args: Record<string, unknown>) => {+ calls.push({ method, args });+ if (method === "conversations.replies" && opts.repliesError) {+ throw Object.assign(new Error(opts.repliesError), { data: { error: opts.repliesError } });+ }+ return typeof result === "function" ? result(args) : result;+ };+ const client = {+ conversations: {+ replies: record("conversations.replies", () => ({ messages: opts.thread ?? [] })),+ history: record("conversations.history", () => ({ messages: opts.thread ?? [] })),+ },+ reactions: {+ get: record("reactions.get", { message: { reactions: opts.reactions ?? [] } }),+ add: record("reactions.add"),+ },+ pins: {+ list: record("pins.list", { items: (opts.pinned ?? []).map((ts) => ({ message: { ts } })) }),+ add: record("pins.add"),+ remove: record("pins.remove"),+ },+ chat: {+ postMessage: record("chat.postMessage", { ts: "900.100" }),+ update: record("chat.update"),+ delete: record("chat.delete"),+ },+ };+ const acknowledgements: string[] = [];+ const checkpoints: Array<[string, string]> = [];+ const queue: unknown[] = [];+ const core = {+ reportRunEditRef: async (runId: string, editRef: string) => {+ checkpoints.push([runId, editRef]);+ },+ holdDeliveryDispatch: (fn: (lost: Promise<void>) => Promise<unknown>) => fn(new Promise<void>(() => {})),+ claimDeliveries: async () => queue.splice(0),+ ackDelivery: async (id: string) => {+ acknowledgements.push(id);+ },+ };+ const poller = createDeliveryPoller({+ core: core as never,+ flow: { inFlightRuns: new Set<string>() } as never,+ threads: { mark: () => {} } as never,+ clientForIdentity: () => client,+ });+ return {+ calls,+ acknowledgements,+ checkpoints,+ methods: () => calls.map((c) => c.method),+ deliver: async (row: {+ destination: Record<string, unknown>;+ idempotencyKey?: string;+ text?: string;+ ageMs?: number;+ }) => {+ queue.push({+ id: "D1",+ idempotencyKey: row.idempotencyKey ?? "post:k1",+ text: row.text ?? "",+ destination: { type: "slack", target: "C1:100.200", ...row.destination },+ createdAt: Date.now() - (row.ageMs ?? 0),+ });+ await poller.pollDeliveries(client);+ },+ };+}++test("a recovered run reply adopts the status placeholder it finds by metadata instead of posting beside it", async () => {+ const h = harness({+ thread: [+ { ts: "100.200", text: "the question" },+ {+ ts: "100.300",+ text: "Working on it…",+ metadata: { event_type: "qm_delivery", event_payload: { idempotency_key: statusPlaceholderKey("r1") } },+ },+ ],+ });+ await h.deliver({ destination: {}, idempotencyKey: "run:r1", text: "the answer", ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(h.methods(), ["conversations.replies", "chat.update"]);+ const update = h.calls[1]!.args;+ assert.equal(update.ts, "100.300");+ assert.equal(update.text, "the answer");+ assert.deepEqual(update.metadata, { event_type: "qm_delivery", event_payload: { idempotency_key: "run:r1" } });+ assert.deepEqual(+ h.checkpoints,+ [["r1", "100.300"]],+ "the adopted placeholder is checkpointed before it is finalized",+ );+ assert.deepEqual(h.acknowledgements, ["D1"]);+});++test("a recovered run reply with no placeholder in the thread posts normally after the probe", async () => {+ const h = harness({ thread: [{ ts: "100.200", text: "the question" }] });+ await h.deliver({ destination: {}, idempotencyKey: "run:r1", text: "the answer", ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(h.methods(), ["conversations.replies", "conversations.replies", "chat.postMessage"]);+ assert.equal(h.calls[2]!.args.text, "the answer");+ assert.deepEqual(h.checkpoints, []);+});++test("a recovered empty run reply deletes the orphaned placeholder it finds by metadata", async () => {+ const h = harness({+ thread: [+ {+ ts: "100.300",+ metadata: { event_type: "qm_delivery", event_payload: { idempotency_key: statusPlaceholderKey("r1") } },+ },+ ],+ });+ await h.deliver({ destination: {}, idempotencyKey: "run:r1", text: "", ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(h.methods(), ["conversations.replies", "chat.delete"]);+ assert.equal(h.calls[1]!.args.ts, "100.300");+});++test("a recorded editRef skips the placeholder probe", async () => {+ const h = harness({});+ await h.deliver({+ destination: { editRef: "100.300" },+ idempotencyKey: "run:r1",+ text: "the answer",+ ageMs: RECOVERED_AGE_MS,+ });+ assert.deepEqual(h.methods(), ["chat.update"]);+});++test("a recovered reaction row is a no-op when Slack already shows the reaction", async () => {+ const h = harness({ reactions: [{ name: "eyes", count: 1 }] });+ await h.deliver({ destination: { react: { messageTs: "100.200", emoji: "eyes" } }, ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(h.methods(), ["reactions.get"]);+ assert.deepEqual(h.acknowledgements, ["D1"]);+});++test("a recovered reaction row still reacts when Slack lacks the reaction", async () => {+ const h = harness({ reactions: [{ name: "tada", count: 1 }] });+ await h.deliver({ destination: { react: { messageTs: "100.200", emoji: "eyes" } }, ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(h.methods(), ["reactions.get", "reactions.add"]);+});++test("a fresh reaction row reacts without probing", async () => {+ const h = harness({ reactions: [{ name: "eyes", count: 1 }] });+ await h.deliver({ destination: { react: { messageTs: "100.200", emoji: "eyes" } } });+ assert.deepEqual(h.methods(), ["reactions.add"]);+});++test("recovered pin and unpin rows are no-ops when the message is already in the requested state", async () => {+ const pinned = harness({ pinned: ["100.200"] });+ await pinned.deliver({ destination: { pin: { messageTs: "100.200" } }, ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(pinned.methods(), ["pins.list"]);+ assert.deepEqual(pinned.acknowledgements, ["D1"]);++ const unpinned = harness({ pinned: [] });+ await unpinned.deliver({ destination: { pin: { messageTs: "100.200", remove: true } }, ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(unpinned.methods(), ["pins.list"]);+});++test("recovered pin and unpin rows act when Slack disagrees with the requested state", async () => {+ const pin = harness({ pinned: [] });+ await pin.deliver({ destination: { pin: { messageTs: "100.200" } }, ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(pin.methods(), ["pins.list", "pins.add"]);++ const unpin = harness({ pinned: ["100.200"] });+ await unpin.deliver({ destination: { pin: { messageTs: "100.200", remove: true } }, ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(unpin.methods(), ["pins.list", "pins.remove"]);+});++test("a recovered delete row is a no-op once the message is gone", async () => {+ const gone = harness({ thread: [] });+ await gone.deliver({ destination: { delete: { messageTs: "100.500" } }, ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(gone.methods(), ["conversations.replies"]);+ assert.deepEqual(gone.acknowledgements, ["D1"]);++ const missingThread = harness({ repliesError: "thread_not_found" });+ await missingThread.deliver({ destination: { delete: { messageTs: "100.500" } }, ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(missingThread.methods(), ["conversations.replies"]);+});++test("a recovered delete row deletes when the message still exists, and a fresh row deletes without probing", async () => {+ const present = harness({ thread: [{ ts: "100.500", text: "to remove" }] });+ await present.deliver({ destination: { delete: { messageTs: "100.500" } }, ageMs: RECOVERED_AGE_MS });+ assert.deepEqual(present.methods(), ["conversations.replies", "chat.delete"]);+ assert.equal(present.calls[0]!.args.ts, "100.500");++ const fresh = harness({ thread: [] });+ await fresh.deliver({ destination: { delete: { messageTs: "100.500" } } });+ assert.deepEqual(fresh.methods(), ["chat.delete"]);+});diff --git a/test/slack-unprompted-ack.test.ts b/test/slack-unprompted-ack.test.tsindex 0118dd617..f4cced04b 100644--- a/test/slack-unprompted-ack.test.ts+++ b/test/slack-unprompted-ack.test.ts@@ -14,6 +14,7 @@ const directory = {function harness(run: (hooks: any) => Promise<unknown>) {const reactions: string[] = [];const posts: string[] = [];+ const postArgs: Array<Record<string, unknown>> = [];const turns: any[] = [];const client = {reactions: {@@ -25,14 +26,21 @@ function harness(run: (hooks: any) => Promise<unknown>) {},},chat: {- postMessage: async ({ text }: { text: string }) => {- posts.push(text);+ postMessage: async (args: Record<string, unknown>) => {+ posts.push(args.text as string);+ postArgs.push(args);return { ts: `9${posts.length}.0` };},+ update: async () => ({}),+ delete: async () => ({}),},};const handler = createTurnHandler({- core: { stageBlob: async () => ({ blobId: "b", sizeBytes: 0 }) },+ core: {+ stageBlob: async () => ({ blobId: "b", sizeBytes: 0 }),+ reportRunEditRef: async () => {},+ reportTurnMetrics: async () => {},+ },flow: {inFlightRuns: { add() {}, delete() {}, has: () => false },inFlightRunByThread: { get() {}, set() {}, clear() {} },@@ -74,9 +82,23 @@ function harness(run: (hooks: any) => Promise<unknown>) {unprompted: true,prefetched: { actor, info: { id: "C1", name: "eng" } as any, audience: [actor] },};- return { handler, client, followup, reactions, posts, turns };+ return { handler, client, followup, reactions, posts, postArgs, turns };}+test("the task-list placeholder is posted with its run's status key so a recovered reply can find it", async () => {+ const h = harness(async (hooks) => {+ await hooks.onQueued?.("r1");+ await hooks.onTasks?.([{ id: "t1", title: "Check staging", status: "in_progress" }]);+ return { status: "ok", reply: "Staging is green." };+ });+ await h.handler.handleIncoming({ ...h.followup, unprompted: false, rawText: "<@UBOT> check staging" }, h.client);+ const placeholder = h.postArgs.find((args) => String(args.text).includes("Check staging"));+ assert.deepEqual(placeholder?.metadata, {+ event_type: "qm_delivery",+ event_payload: { idempotency_key: "status:run:r1" },+ });+});+test("an unmentioned thread followup QM answers gets the ack reaction added then cleared", async () => {const h = harness(async (hooks) => {hooks.onReplying?.();