Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Verify Slack state before repeating cut deliveries and sweep silent terminal runs (#2053)

Regan Bell93c64f516b88
diff --git a/src/delivery/delivery-store.ts b/src/delivery/delivery-store.ts
index 39d22f4f3..a81afbe74 100644
--- a/src/delivery/delivery-store.ts
+++ b/src/delivery/delivery-store.ts
@@ -50,6 +50,7 @@ export interface DeliveryStore {
ackByKey(idempotencyKey: string, at: number): Promise<void>;
setEditRefByKey(idempotencyKey: string, editRef: string): Promise<void>;
get(id: string): Promise<Delivery | null>;
+ existingKeys(idempotencyKeys: string[]): Promise<Set<string>>;
recordRecipientThread(id: string, recipientThreadRef: string, at: number): Promise<void>;
listByRecipientThread(recipientThreadRef: string, opts?: { limit?: number }): Promise<Delivery[]>;
listBySourceSession(sourceSessionId: string, sourceThreadRef: string, opts?: { limit?: number }): Promise<Delivery[]>;
@@ -176,6 +177,9 @@ export function createDeliveryStore(opts?: { maxAgeMs?: number }): DeliveryStore
async get(id) {
return deliveries.get(id) ?? null;
},
+ async existingKeys(idempotencyKeys) {
+ return new Set(idempotencyKeys.filter((key) => byKey.has(key)));
+ },
async recordRecipientThread(id, recipientThreadRef, at) {
const d = deliveries.get(id);
if (!d || d.destination.type !== "principal") return;
diff --git a/src/delivery/postgres-delivery-store.ts b/src/delivery/postgres-delivery-store.ts
index 5fa46ffbf..daef5355a 100644
--- a/src/delivery/postgres-delivery-store.ts
+++ b/src/delivery/postgres-delivery-store.ts
@@ -218,6 +218,13 @@ export function createPostgresDeliveryStore(connectionString: string, opts?: { m
const rows = await q("SELECT * FROM deliveries WHERE id = $1", [id]);
return rows[0] ? rowToDelivery(rows[0]) : null;
},
+ async existingKeys(idempotencyKeys) {
+ if (!idempotencyKeys.length) return new Set();
+ const rows = await q("SELECT idempotency_key FROM deliveries WHERE idempotency_key = ANY($1::text[])", [
+ idempotencyKeys,
+ ]);
+ return new Set(rows.map((r) => r.idempotency_key as string));
+ },
async recordRecipientThread(id, recipientThreadRef, at) {
await query(
`UPDATE deliveries
diff --git a/src/delivery/run-result-delivery.ts b/src/delivery/run-result-delivery.ts
index 171989a0b..ec060fcd0 100644
--- a/src/delivery/run-result-delivery.ts
+++ b/src/delivery/run-result-delivery.ts
@@ -138,24 +138,53 @@ export async function recordRunFailureEntry(sessions: TurnFailureSessions, run:
}
}
+const RUN_RESULT_SWEEP_LOOKBACK_MS = 60 * 60_000;
+const RUN_RESULT_SWEEP_GRACE_MS = 60_000;
+const RUN_RESULT_SWEEP_PAGE = 200;
+
+export interface RunResultRecovery {
+ sweep(now?: number): Promise<number>;
+}
+
export function wireRunResultDeliveries(
- runs: RunStore,
- deliveries: DeliveryStore,
+ runs: Pick<RunStore, "onTerminal" | "terminalFinished">,
+ deliveries: Pick<DeliveryStore, "enqueue" | "existingKeys">,
tasks?: TaskStore,
adminUrlFor?: AdminUrlFor,
sessions?: TurnFailureSessions,
-): void {
- runs.onTerminal((run) => {
+): RunResultRecovery {
+ const recover = async (run: Run): Promise<boolean> => {
if (sessions) {
void recordRunFailureEntry(sessions, run).catch(
reportFailureAs("delivery: record turn_failure entry", undefined, `run=${run.id}`),
);
}
- void (async () => {
- const taskList = tasks ? await tasks.list({ originRunId: run.id }) : [];
- const delivery = runResultDelivery(run, taskList, adminUrlFor);
- if (!delivery) return;
- await deliveries.enqueue(delivery);
- })().catch(reportFailureAs("delivery: enqueue recovery delivery", undefined, `run=${run.id}`));
+ const taskList = tasks ? await tasks.list({ originRunId: run.id }) : [];
+ const delivery = runResultDelivery(run, taskList, adminUrlFor);
+ if (!delivery) return false;
+ await deliveries.enqueue(delivery);
+ return true;
+ };
+ runs.onTerminal((run) => {
+ void recover(run).catch(reportFailureAs("delivery: enqueue recovery delivery", undefined, `run=${run.id}`));
});
+ return {
+ async sweep(now = Date.now()) {
+ const before = now - RUN_RESULT_SWEEP_GRACE_MS;
+ let cursor = { finishedAt: now - RUN_RESULT_SWEEP_LOOKBACK_MS, id: "" };
+ let recovered = 0;
+ for (;;) {
+ const batch = await runs.terminalFinished(cursor, before, RUN_RESULT_SWEEP_PAGE);
+ const existing = await deliveries.existingKeys(batch.map((run) => `run:${run.id}`));
+ for (const run of batch) {
+ if (existing.has(`run:${run.id}`) || !(await recover(run))) continue;
+ console.error(`[delivery] terminal run ${run.id} had no outbox row — enqueued its recovery delivery`);
+ recovered += 1;
+ }
+ const last = batch.at(-1);
+ if (!last || batch.length < RUN_RESULT_SWEEP_PAGE) return recovered;
+ cursor = { finishedAt: last.finishedAt!, id: last.id };
+ }
+ },
+ };
}
diff --git a/src/runs/memory-run-store.ts b/src/runs/memory-run-store.ts
index 88ff6eb43..c49b78926 100644
--- a/src/runs/memory-run-store.ts
+++ b/src/runs/memory-run-store.ts
@@ -186,6 +186,18 @@ export function createMemoryRunStore(opts?: { maxClaims?: number }): MemoryRunti
.sort((a, b) => a.id.localeCompare(b.id))
.slice(0, limit);
},
+ async terminalFinished(after, beforeMs, limit) {
+ return [...runs.values()]
+ .filter(
+ (run) =>
+ isTerminal(run.status) &&
+ run.finishedAt !== null &&
+ run.finishedAt <= beforeMs &&
+ (run.finishedAt > after.finishedAt || (run.finishedAt === after.finishedAt && run.id > after.id)),
+ )
+ .sort((a, b) => a.finishedAt! - b.finishedAt! || (a.id < b.id ? -1 : 1))
+ .slice(0, limit);
+ },
async markReturned(runId) {
returned.add(runId);
},
diff --git a/src/runs/postgres-run-store.ts b/src/runs/postgres-run-store.ts
index 4273fa53e..4824f9ab0 100644
--- a/src/runs/postgres-run-store.ts
+++ b/src/runs/postgres-run-store.ts
@@ -126,6 +126,12 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
`CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_created ON runs(created_at DESC)`,
],
},
+ {
+ id: "runs/store/0006-terminal-finished",
+ statements: [
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_runs_terminal_finished ON runs(finished_at, id) WHERE status IN ('done','failed')`,
+ ],
+ },
],
[
{
@@ -401,6 +407,15 @@ export function createPostgresRunStore(connectionString: string, opts?: { maxCla
);
return rows.map(rowToRun);
},
+ async terminalFinished(after, beforeMs, limit) {
+ const { rows } = await q(
+ `SELECT * FROM runs WHERE status IN ('done','failed') AND finished_at <= $1
+ AND (finished_at > $2 OR (finished_at = $2 AND id > $3))
+ ORDER BY finished_at, id LIMIT $4`,
+ [beforeMs, after.finishedAt, after.id, limit],
+ );
+ return rows.map(rowToRun);
+ },
async markReturned(runId) {
await q("UPDATE runs SET returned_at = $2 WHERE id = $1 AND status IN ('done','failed')", [runId, Date.now()]);
},
diff --git a/src/runs/run-store.ts b/src/runs/run-store.ts
index 347c1425a..fcc328196 100644
--- a/src/runs/run-store.ts
+++ b/src/runs/run-store.ts
@@ -39,6 +39,11 @@ export interface Run {
finishedAt: number | null;
}
+interface TerminalCursor {
+ finishedAt: number;
+ id: string;
+}
+
export interface EnqueueInput {
sessionId: string;
request: OrchestratorInput;
@@ -84,6 +89,7 @@ export interface RunStore {
latestForThread(threadRef: string, opts?: { excludePrivateMessages?: boolean }): Promise<Run | null>;
pendingReturns(limit?: number, afterId?: string): Promise<Run[]>;
+ terminalFinished(after: TerminalCursor, beforeMs: number, limit: number): Promise<Run[]>;
markReturned(runId: string): Promise<void>;
deferReturn(runId: string, delayMs: number): Promise<void>;
diff --git a/src/slack/deliveries.ts b/src/slack/deliveries.ts
index deeeafba4..d3891b57d 100644
--- a/src/slack/deliveries.ts
+++ b/src/slack/deliveries.ts
@@ -11,10 +11,15 @@ import {
createDeliveryTracker,
createThreadTracker,
deliverWithRetry,
+ deliveryMetadata,
dmThreadRef,
openConversationFor,
findPostedByKey,
+ messageExists,
+ messagePinned,
parseDeliveryTarget,
+ reactionPresent,
+ statusPlaceholderKey,
postWithVerify,
recoveryVerifyOldest,
renderTaskList,
@@ -47,6 +52,7 @@ const PERMANENT_POST_ERRORS = new Set([
const DELIVERY_CLAIM_MARGIN_MS = 2_000;
const RUN_RECOVERY_GRACE_MS = 15_000;
+const STATUS_PLACEHOLDER_LOOKBACK_MS = 6 * 3_600_000;
function mergeSlackApiMs(body: unknown, slackApiMs: number | undefined): unknown {
if (slackApiMs === undefined) return body;
@@ -236,10 +242,11 @@ export function createDeliveryPoller(deps: {
await deliverKeychainCard(core, client, d, channel, threadTs, deps.webUiPublicUrl);
return undefined;
}
+ const recovered = recoveredRow(d);
if (d.destination.react) {
- const { failed } = await applyReactions(client, channel, d.destination.react.messageTs, [
- d.destination.react.emoji,
- ]);
+ const { messageTs, emoji } = d.destination.react;
+ if (recovered && (await reactionPresent(client, channel, messageTs, emoji))) return undefined;
+ const { failed } = await applyReactions(client, channel, messageTs, [emoji]);
if (failed.length)
console.error(
`[slack-plugin] delivery ${d.id} reaction(s) failed: ${failed.join(", ")} (check reactions:write / message ts)`,
@@ -248,6 +255,7 @@ export function createDeliveryPoller(deps: {
}
if (d.destination.pin) {
const { messageTs, remove } = d.destination.pin;
+ if (recovered && (await messagePinned(client, channel, messageTs)) === !remove) return undefined;
try {
if (remove) await client.pins.remove({ channel, timestamp: messageTs });
else await client.pins.add({ channel, timestamp: messageTs });
@@ -261,8 +269,10 @@ export function createDeliveryPoller(deps: {
return undefined;
}
if (d.destination.delete) {
+ const { messageTs } = d.destination.delete;
+ if (recovered && (await messageExists(client, channel, messageTs)) === false) return undefined;
try {
- await client.chat.delete({ channel, ts: d.destination.delete.messageTs });
+ await client.chat.delete({ channel, ts: messageTs });
} catch (err) {
console.error(
`[slack-plugin] delivery ${d.id} delete failed: ${slackErrorCode(err) ?? (err as Error).message} (own messages only)`,
@@ -270,6 +280,19 @@ export function createDeliveryPoller(deps: {
}
return undefined;
}
+ let editRef = d.destination.editRef;
+ if (!editRef && runId) {
+ const placeholder = await findPostedByKey(
+ client,
+ { channel, ...(threadTs ? { thread_ts: threadTs } : {}) },
+ statusPlaceholderKey(runId),
+ String((d.createdAt - STATUS_PLACEHOLDER_LOOKBACK_MS) / 1000),
+ ).catch(swallowAs("slack: status-placeholder probe", undefined));
+ if (placeholder) {
+ await core.reportRunEditRef(runId, placeholder.ts);
+ editRef = placeholder.ts;
+ }
+ }
let text = toSlackMrkdwn(runId ? cleanAgentReplyForSlack(d.text).text : stripSlackDirectives(d.text));
const replayAttachments = async (root?: string): Promise<void> => {
if (!d.attachments?.length) return;
@@ -304,11 +327,11 @@ export function createDeliveryPoller(deps: {
if (!text.trim() && !(messageFooter.length && d.attachments?.length)) {
if (taskList) {
let preserved = false;
- if (d.destination.editRef) {
+ if (editRef) {
try {
await client.chat.update({
channel,
- ts: d.destination.editRef,
+ ts: editRef,
text: taskList,
blocks: [{ type: "section", text: { type: "mrkdwn", text: taskList } }],
...botIdentityArgs(),
@@ -324,9 +347,9 @@ export function createDeliveryPoller(deps: {
blocks: [{ type: "section", text: { type: "mrkdwn", text: taskList } }],
});
}
- } else if (d.destination.editRef) {
+ } else if (editRef) {
await client.chat
- .delete({ channel, ts: d.destination.editRef })
+ .delete({ channel, ts: editRef })
.catch(swallowAs("slack: delete status placeholder", undefined));
}
await replayAttachments(threadTs);
@@ -335,7 +358,7 @@ export function createDeliveryPoller(deps: {
}
const verifyOldest = recoveryVerifyOldest(
typeof d.createdAt === "number" ? d.createdAt : undefined,
- d.destination.editRef,
+ editRef,
);
const deliveredMarker = (): Promise<{ ts: string; channel: string } | undefined> =>
findPostedByKey(
@@ -344,16 +367,17 @@ export function createDeliveryPoller(deps: {
d.idempotencyKey ?? d.id,
verifyOldest ?? String(Date.now() / 1000 - 60),
).catch(swallowAs("slack: delivered-marker probe", undefined));
- if (d.destination.editRef) {
+ if (editRef) {
const unfurlLinks = runId ? false : d.destination.unfurlLinks;
try {
const alreadyDelivered = d.attachments?.length ? await deliveredMarker() : undefined;
await client.chat.update({
channel,
- ts: d.destination.editRef,
+ ts: editRef,
text,
...(footerBlocks ? { blocks: footerBlocks } : {}),
...botIdentityArgs(),
+ metadata: deliveryMetadata(d.idempotencyKey ?? d.id),
...(unfurlLinks !== undefined ? { unfurl_links: unfurlLinks, unfurl_media: unfurlLinks } : {}),
});
if (threadTs) threads.mark(channel, threadTs, true);
diff --git a/src/slack/delivery.ts b/src/slack/delivery.ts
index 20fb7d7c8..7263a47d3 100644
--- a/src/slack/delivery.ts
+++ b/src/slack/delivery.ts
@@ -470,6 +470,74 @@ export async function findPostedByKey(
return undefined;
}
+export function statusPlaceholderKey(runId: string): string {
+ return `status:run:${runId}`;
+}
+
+interface SlackStateClient {
+ reactions: { get(args: { channel: string; timestamp: string; full: boolean }): Promise<unknown> };
+ pins: { list(args: { channel: string }): Promise<unknown> };
+ conversations: {
+ replies(args: {
+ channel: string;
+ ts: string;
+ oldest: string;
+ latest: string;
+ inclusive: boolean;
+ limit: number;
+ }): Promise<unknown>;
+ };
+}
+
+export async function reactionPresent(
+ client: Pick<SlackStateClient, "reactions">,
+ channel: string,
+ timestamp: string,
+ name: string,
+): Promise<boolean | undefined> {
+ try {
+ const res = (await client.reactions.get({ channel, timestamp, full: true })) as {
+ message?: { reactions?: Array<{ name?: string; count?: number }> };
+ };
+ return (res.message?.reactions ?? []).some((r) => r.name === name && (r.count ?? 0) > 0);
+ } catch {
+ return undefined;
+ }
+}
+
+export async function messagePinned(
+ client: Pick<SlackStateClient, "pins">,
+ channel: string,
+ timestamp: string,
+): Promise<boolean | undefined> {
+ try {
+ const res = (await client.pins.list({ channel })) as { items?: Array<{ message?: { ts?: string } }> };
+ return (res.items ?? []).some((item) => item.message?.ts === timestamp);
+ } catch {
+ return undefined;
+ }
+}
+
+export async function messageExists(
+ client: Pick<SlackStateClient, "conversations">,
+ channel: string,
+ timestamp: string,
+): Promise<boolean | undefined> {
+ try {
+ const res = (await client.conversations.replies({
+ channel,
+ ts: timestamp,
+ oldest: timestamp,
+ latest: timestamp,
+ inclusive: true,
+ limit: 1,
+ })) as { messages?: Array<{ ts?: string }> };
+ return (res.messages ?? []).some((m) => m.ts === timestamp);
+ } catch (err) {
+ return (err as { data?: { error?: string } }).data?.error === "thread_not_found" ? false : undefined;
+ }
+}
+
export function recoveryVerifyOldest(createdAt: number | undefined, editRef: string | undefined): string | undefined {
const bounds: number[] = [];
if (typeof createdAt === "number") bounds.push(createdAt / 1000 - 60);
diff --git a/src/slack/lib.ts b/src/slack/lib.ts
index fb294bed4..815814be6 100644
--- a/src/slack/lib.ts
+++ b/src/slack/lib.ts
@@ -140,6 +140,10 @@ export {
postWithVerify,
findPostedByKey,
recoveryVerifyOldest,
+ statusPlaceholderKey,
+ reactionPresent,
+ messagePinned,
+ messageExists,
deliveryMetadata,
type DeliveryMetadata,
type PostedPart,
diff --git a/src/slack/turn-handler.ts b/src/slack/turn-handler.ts
index cad339348..68c24058a 100644
--- a/src/slack/turn-handler.ts
+++ b/src/slack/turn-handler.ts
@@ -34,6 +34,7 @@ import {
dedupedRun,
deliveryCandidatesFor,
deliveryMetadata,
+ statusPlaceholderKey,
channelThreadRef,
dmThreadRef,
downloadSlackFile,
@@ -395,13 +396,14 @@ export function createTurnHandler(deps: {
}
let ack: AckPresenter | undefined;
+ const statusKey = (): string | undefined => (queuedRunId ? statusPlaceholderKey(queuedRunId) : undefined);
const startAck = (): AckPresenter =>
(ack ??= createAckPresenter({
taskManaged,
postAck: async (text) => {
const rendered = toSlackMrkdwn(text);
if (await taskList?.addLead(rendered)) return;
- const ts = await postReply(rendered);
+ const ts = await postReply(rendered, undefined, statusKey());
if (ts) {
await taskList?.attach(ts, rendered);
}
@@ -424,7 +426,7 @@ export function createTurnHandler(deps: {
if (!inc.unprompted) startAck();
if (!inc.unprompted) {
taskList = createTaskListPresenter({
- post: (text, blocks) => postReply(text, blocks),
+ post: (text, blocks) => postReply(text, blocks, statusKey()),
update: (ts, text, blocks, metadata) =>
client.chat
.update({
diff --git a/src/wiring.ts b/src/wiring.ts
index 424dc2b98..8fb402aba 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -2001,7 +2001,7 @@ export function buildApp(
? (sessionId: string): string | undefined =>
uuidId.test(sessionId) ? adminSessionUrl(recoveryAdminBase, sessionId) : undefined
: undefined;
- wireRunResultDeliveries(runs, deliveries, tasks, recoveryAdminUrlFor, sessions);
+ const runResultRecovery = wireRunResultDeliveries(runs, deliveries, tasks, recoveryAdminUrlFor, sessions);
const idempotency = createIdempotencyStore(artifactMap<IdempotencyRecord>("idempotency"));
const skillFetcher = createGitFetcher(
keychain
@@ -2313,6 +2313,14 @@ export function buildApp(
1_000,
{ label: "session-returns", immediate: true },
);
+ const runResultSweeper = createSweeper(
+ () =>
+ advisoryLock.tryWithLock
+ ? advisoryLock.tryWithLock("run-result-sweep", () => runResultRecovery.sweep())
+ : advisoryLock.withLock("run-result-sweep", () => runResultRecovery.sweep()),
+ 60_000,
+ { label: "run-results" },
+ );
const orphanedSignalSweeper = createSweeper(
async () => {
for (const runId of await runSignals.pendingRunIds()) {
@@ -2669,6 +2677,7 @@ export function buildApp(
swarms?.start();
orphanedSignalSweeper.start();
sessionReturnSweeper.start();
+ runResultSweeper.start();
approvalDeliverySweeper.start();
};
if (backgroundStopping)
@@ -2698,6 +2707,7 @@ export function buildApp(
swarms?.stop(),
orphanedSignalSweeper.stop(),
sessionReturnSweeper.stop(),
+ runResultSweeper.stop(),
approvalDeliverySweeper.stop(),
...workers.map((worker) => worker.stopClaims()),
];
diff --git a/test/delivery-store-contract.ts b/test/delivery-store-contract.ts
index 687d4d146..c46b3a764 100644
--- a/test/delivery-store-contract.ts
+++ b/test/delivery-store-contract.ts
@@ -235,6 +235,13 @@ export async function exerciseDeliveryStore(store: DeliveryStore): Promise<void>
"an expired claim re-surfaces (at-least-once)",
);
await store.ack(abandoned.id, 700);
+
+ assert.deepEqual(
+ [...(await store.existingKeys(["fire-1", "run:r-2", "fire-race-2", "never-enqueued"]))].sort(),
+ ["fire-1", "fire-race-2", "run:r-2"],
+ "existing keys cover enqueued rows and ack tombstones alike",
+ );
+ assert.deepEqual(await store.existingKeys([]), new Set());
}
export async function exerciseDeliveryExpiry(makeStore: (opts: { maxAgeMs: number }) => DeliveryStore): Promise<void> {
diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.ts
index a5d2c2911..9b4bc0d1b 100644
--- a/test/postgres-store.test.ts
+++ b/test/postgres-store.test.ts
@@ -1797,6 +1797,40 @@ test("pg run store: Unicode stays jsonb-safe through enqueue, edit and both stee
}
});
+test("pg run store: terminalFinished lists finished runs by (finishedAt, id) up to the bound", { skip }, async () => {
+ const { runs, close } = createPostgresRunStore(URL!);
+ const tag = randomUUID();
+ try {
+ const ids: string[] = [];
+ for (const ref of ["a", "b", "c"]) {
+ const thread = `${tag}-${ref}`;
+ const run = (await runs.enqueue({ sessionId: thread, request: turn(ref) })).run;
+ const claimed = await runs.claimForSession(thread, "w", 5_000);
+ await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply: ref });
+ ids.push(run.id);
+ }
+ const open = (await runs.enqueue({ sessionId: `${tag}-open`, request: turn("open") })).run;
+ await runs.claimForSession(`${tag}-open`, "w", 5_000);
+ const far = Date.now() + 60_000;
+ const started = (await runs.get(ids[0]!))!.startedAt! - 1;
+ const listed = await runs.terminalFinished({ finishedAt: started, id: "" }, far, 10_000);
+ const mine = listed.filter((run) => run.sessionId.startsWith(tag));
+ assert.deepEqual(new Set(mine.map((run) => run.id)), new Set(ids));
+ assert.ok(!listed.some((run) => run.id === open.id));
+ for (let i = 1; i < listed.length; i++) {
+ const prev = listed[i - 1]!;
+ const next = listed[i]!;
+ assert.ok(prev.finishedAt! < next.finishedAt! || (prev.finishedAt === next.finishedAt && prev.id < next.id));
+ }
+ const first = mine[0]!;
+ const after = await runs.terminalFinished({ finishedAt: first.finishedAt!, id: first.id }, far, 10_000);
+ assert.ok(!after.some((run) => run.id === first.id), "the cursor row itself is excluded");
+ assert.deepEqual(await runs.terminalFinished({ finishedAt: started, id: "" }, first.finishedAt! - 1, 10_000), []);
+ } finally {
+ await close();
+ }
+});
+
test("pg run store: a session_busy completion frees the dedup key so the same key runs again", { skip }, async () => {
const { runs, close } = createPostgresRunStore(URL!);
try {
diff --git a/test/run-result-delivery.test.ts b/test/run-result-delivery.test.ts
index 133e7ce9a..ca1534bd4 100644
--- a/test/run-result-delivery.test.ts
+++ b/test/run-result-delivery.test.ts
@@ -255,6 +255,80 @@ test("wired stores: a completed turn lands in the outbox unless the live path ac
);
});
+test("sweep enqueues the recovery delivery for a terminal run whose outbox row never landed", async () => {
+ const { runs } = createMemoryRunStore();
+ const deliveries = createDeliveryStore();
+ const bypassed = (await runs.enqueue({ sessionId: "sA", request: turn("a", "C9:171.001") })).run;
+ const claimed = await runs.claim("w1", 5_000);
+ await runs.complete(bypassed.id, claimed?.leaseToken ?? "", { status: "ok", reply: "lost on the way out" });
+ const recovery = wireRunResultDeliveries(runs, deliveries);
+ assert.deepEqual(await deliveries.pending("slack"), [], "wired after the fact: nothing enqueued");
+
+ assert.equal(await recovery.sweep(Date.now() + 120_000), 1);
+ const pending = await deliveries.pending("slack");
+ assert.deepEqual(
+ pending.map((d) => [d.idempotencyKey, d.text]),
+ [[`run:${bypassed.id}`, "lost on the way out"]],
+ );
+ assert.equal(await recovery.sweep(Date.now() + 120_000), 0, "a second sweep finds the row and leaves it alone");
+});
+
+test("sweep leaves runs alone when their delivery already exists, was acked live, or is still inside the grace window", async () => {
+ const { runs } = createMemoryRunStore();
+ const deliveries = createDeliveryStore();
+ const recovery = wireRunResultDeliveries(runs, deliveries);
+ const finish = async (threadRef: string, reply: string): Promise<Run> => {
+ const run = (await runs.enqueue({ sessionId: threadRef, request: turn(reply, "C9:171.001") })).run;
+ const claimed = await runs.claimForSession(threadRef, "w1", 5_000);
+ await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply });
+ return run;
+ };
+ const enqueued = await finish("sA", "already enqueued");
+ const live = await finish("sB", "delivered live");
+ await deliveries.ackByKey(`run:${live.id}`, Date.now());
+ const silent = (await runs.enqueue({ sessionId: "sC", request: turn("c", "C9:171.001") })).run;
+ await runs.complete(silent.id, (await runs.claimForSession("sC", "w1", 5_000))?.leaseToken ?? "", {
+ status: "silent",
+ });
+ await new Promise((r) => setTimeout(r, 0));
+ assert.deepEqual(
+ (await deliveries.pending("slack")).map((d) => d.idempotencyKey),
+ [`run:${enqueued.id}`],
+ );
+
+ assert.equal(await recovery.sweep(Date.now() + 120_000), 0);
+ assert.deepEqual(
+ (await deliveries.pending("slack")).map((d) => d.idempotencyKey),
+ [`run:${enqueued.id}`],
+ );
+
+ const fresh = await finish("sD", "just finished");
+ await deliveries.ackByKey(`run:${fresh.id}`, Date.now());
+ const bare = createDeliveryStore();
+ const bareRecovery = wireRunResultDeliveries(runs, bare);
+ assert.equal(await bareRecovery.sweep(Date.now()), 0, "runs finished inside the grace window are not swept yet");
+ assert.equal(await bareRecovery.sweep(Date.now() + 120_000), 3, "the same runs sweep once the grace window passes");
+});
+
+test("sweep pages through terminal runs by (finishedAt, id) without skipping ties", async () => {
+ const { runs } = createMemoryRunStore();
+ const deliveries = createDeliveryStore();
+ const recovery = wireRunResultDeliveries(runs, deliveries);
+ const finished = new Set<string>();
+ for (let i = 0; i < 205; i++) {
+ const run = (await runs.enqueue({ sessionId: `s${i}`, request: turn(`r${i}`, "C9:171.001") })).run;
+ const claimed = await runs.claimForSession(`s${i}`, "w1", 5_000);
+ await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply: `r${i}` });
+ finished.add(`run:${run.id}`);
+ }
+ await new Promise((r) => setTimeout(r, 0));
+ const swept = createDeliveryStore();
+ const sweptRecovery = wireRunResultDeliveries(runs, swept);
+ assert.equal(await sweptRecovery.sweep(Date.now() + 120_000), 205);
+ assert.deepEqual(new Set((await swept.pending("slack")).map((d) => d.idempotencyKey)), finished);
+ assert.equal(await recovery.sweep(Date.now() + 120_000), 0);
+});
+
test("wired stores: a parked run lands a durable, non-ackable failure note", async () => {
const { runs } = createMemoryRunStore();
const deliveries = createDeliveryStore();
diff --git a/test/run-store.test.ts b/test/run-store.test.ts
index 81a0f5236..53abb9ac0 100644
--- a/test/run-store.test.ts
+++ b/test/run-store.test.ts
@@ -35,6 +35,37 @@ for (const backend of backends) {
]);
});
+ test(`[${backend.name}] terminalFinished pages finished runs by (finishedAt, id) within the bound`, async () => {
+ const { runs } = backend.make();
+ const finished: string[] = [];
+ for (const ref of ["a", "b", "c"]) {
+ const run = (await runs.enqueue({ sessionId: ref, request: turn(ref) })).run;
+ const claimed = await runs.claimForSession(ref, "w", 5_000);
+ await runs.complete(run.id, claimed?.leaseToken ?? "", { status: "ok", reply: ref });
+ finished.push(run.id);
+ }
+ await runs.enqueue({ sessionId: "pending", request: turn("pending") });
+ const running = (await runs.enqueue({ sessionId: "running", request: turn("running") })).run;
+ await runs.claimForSession("running", "w", 5_000);
+ const far = Date.now() + 60_000;
+ const all = await runs.terminalFinished({ finishedAt: 0, id: "" }, far, 10);
+ assert.deepEqual(new Set(all.map((run) => run.id)), new Set(finished), "only terminal runs are listed");
+ assert.ok(!all.some((run) => run.id === running.id));
+ for (let i = 1; i < all.length; i++) {
+ const prev = all[i - 1]!;
+ const next = all[i]!;
+ assert.ok(prev.finishedAt! < next.finishedAt! || (prev.finishedAt === next.finishedAt && prev.id < next.id));
+ }
+ const first = await runs.terminalFinished({ finishedAt: 0, id: "" }, far, 2);
+ const rest = await runs.terminalFinished({ finishedAt: first[1]!.finishedAt!, id: first[1]!.id }, far, 2);
+ assert.deepEqual(
+ [...first, ...rest].map((run) => run.id),
+ all.map((run) => run.id),
+ "keyset paging is gapless",
+ );
+ assert.deepEqual(await runs.terminalFinished({ finishedAt: 0, id: "" }, all[0]!.finishedAt! - 1, 10), []);
+ });
+
test(`[${backend.name}] enqueue dedups by dedup key`, async () => {
const { runs } = backend.make();
const a = await runs.enqueue({ sessionId: "s1", request: turn("hi"), dedupKey: "k1" });
diff --git a/test/slack-delivery-recovery.test.ts b/test/slack-delivery-recovery.test.ts
new file mode 100644
index 000000000..91c26d01b
--- /dev/null
+++ b/test/slack-delivery-recovery.test.ts
@@ -0,0 +1,204 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { createDeliveryPoller } from "../src/slack/deliveries.ts";
+import { statusPlaceholderKey } from "../src/slack/lib.ts";
+
+const RECOVERED_AGE_MS = 60_000;
+
+function harness(opts: {
+ thread?: Array<Record<string, unknown>>;
+ reactions?: Array<{ name: string; count: number }>;
+ pinned?: string[];
+ repliesError?: string;
+}) {
+ const calls: Array<{ method: string; args: Record<string, unknown> }> = [];
+ const record =
+ (method: string, result: unknown = {}) =>
+ async (args: Record<string, unknown>) => {
+ calls.push({ method, args });
+ if (method === "conversations.replies" && opts.repliesError) {
+ throw Object.assign(new Error(opts.repliesError), { data: { error: opts.repliesError } });
+ }
+ return typeof result === "function" ? result(args) : result;
+ };
+ const client = {
+ conversations: {
+ replies: record("conversations.replies", () => ({ messages: opts.thread ?? [] })),
+ history: record("conversations.history", () => ({ messages: opts.thread ?? [] })),
+ },
+ reactions: {
+ get: record("reactions.get", { message: { reactions: opts.reactions ?? [] } }),
+ add: record("reactions.add"),
+ },
+ pins: {
+ list: record("pins.list", { items: (opts.pinned ?? []).map((ts) => ({ message: { ts } })) }),
+ add: record("pins.add"),
+ remove: record("pins.remove"),
+ },
+ chat: {
+ postMessage: record("chat.postMessage", { ts: "900.100" }),
+ update: record("chat.update"),
+ delete: record("chat.delete"),
+ },
+ };
+ const acknowledgements: string[] = [];
+ const checkpoints: Array<[string, string]> = [];
+ const queue: unknown[] = [];
+ const core = {
+ reportRunEditRef: async (runId: string, editRef: string) => {
+ checkpoints.push([runId, editRef]);
+ },
+ holdDeliveryDispatch: (fn: (lost: Promise<void>) => Promise<unknown>) => fn(new Promise<void>(() => {})),
+ claimDeliveries: async () => queue.splice(0),
+ ackDelivery: async (id: string) => {
+ acknowledgements.push(id);
+ },
+ };
+ const poller = createDeliveryPoller({
+ core: core as never,
+ flow: { inFlightRuns: new Set<string>() } as never,
+ threads: { mark: () => {} } as never,
+ clientForIdentity: () => client,
+ });
+ return {
+ calls,
+ acknowledgements,
+ checkpoints,
+ methods: () => calls.map((c) => c.method),
+ deliver: async (row: {
+ destination: Record<string, unknown>;
+ idempotencyKey?: string;
+ text?: string;
+ ageMs?: number;
+ }) => {
+ queue.push({
+ id: "D1",
+ idempotencyKey: row.idempotencyKey ?? "post:k1",
+ text: row.text ?? "",
+ destination: { type: "slack", target: "C1:100.200", ...row.destination },
+ createdAt: Date.now() - (row.ageMs ?? 0),
+ });
+ await poller.pollDeliveries(client);
+ },
+ };
+}
+
+test("a recovered run reply adopts the status placeholder it finds by metadata instead of posting beside it", async () => {
+ const h = harness({
+ thread: [
+ { ts: "100.200", text: "the question" },
+ {
+ ts: "100.300",
+ text: "Working on it…",
+ metadata: { event_type: "qm_delivery", event_payload: { idempotency_key: statusPlaceholderKey("r1") } },
+ },
+ ],
+ });
+ await h.deliver({ destination: {}, idempotencyKey: "run:r1", text: "the answer", ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(h.methods(), ["conversations.replies", "chat.update"]);
+ const update = h.calls[1]!.args;
+ assert.equal(update.ts, "100.300");
+ assert.equal(update.text, "the answer");
+ assert.deepEqual(update.metadata, { event_type: "qm_delivery", event_payload: { idempotency_key: "run:r1" } });
+ assert.deepEqual(
+ h.checkpoints,
+ [["r1", "100.300"]],
+ "the adopted placeholder is checkpointed before it is finalized",
+ );
+ assert.deepEqual(h.acknowledgements, ["D1"]);
+});
+
+test("a recovered run reply with no placeholder in the thread posts normally after the probe", async () => {
+ const h = harness({ thread: [{ ts: "100.200", text: "the question" }] });
+ await h.deliver({ destination: {}, idempotencyKey: "run:r1", text: "the answer", ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(h.methods(), ["conversations.replies", "conversations.replies", "chat.postMessage"]);
+ assert.equal(h.calls[2]!.args.text, "the answer");
+ assert.deepEqual(h.checkpoints, []);
+});
+
+test("a recovered empty run reply deletes the orphaned placeholder it finds by metadata", async () => {
+ const h = harness({
+ thread: [
+ {
+ ts: "100.300",
+ metadata: { event_type: "qm_delivery", event_payload: { idempotency_key: statusPlaceholderKey("r1") } },
+ },
+ ],
+ });
+ await h.deliver({ destination: {}, idempotencyKey: "run:r1", text: "", ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(h.methods(), ["conversations.replies", "chat.delete"]);
+ assert.equal(h.calls[1]!.args.ts, "100.300");
+});
+
+test("a recorded editRef skips the placeholder probe", async () => {
+ const h = harness({});
+ await h.deliver({
+ destination: { editRef: "100.300" },
+ idempotencyKey: "run:r1",
+ text: "the answer",
+ ageMs: RECOVERED_AGE_MS,
+ });
+ assert.deepEqual(h.methods(), ["chat.update"]);
+});
+
+test("a recovered reaction row is a no-op when Slack already shows the reaction", async () => {
+ const h = harness({ reactions: [{ name: "eyes", count: 1 }] });
+ await h.deliver({ destination: { react: { messageTs: "100.200", emoji: "eyes" } }, ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(h.methods(), ["reactions.get"]);
+ assert.deepEqual(h.acknowledgements, ["D1"]);
+});
+
+test("a recovered reaction row still reacts when Slack lacks the reaction", async () => {
+ const h = harness({ reactions: [{ name: "tada", count: 1 }] });
+ await h.deliver({ destination: { react: { messageTs: "100.200", emoji: "eyes" } }, ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(h.methods(), ["reactions.get", "reactions.add"]);
+});
+
+test("a fresh reaction row reacts without probing", async () => {
+ const h = harness({ reactions: [{ name: "eyes", count: 1 }] });
+ await h.deliver({ destination: { react: { messageTs: "100.200", emoji: "eyes" } } });
+ assert.deepEqual(h.methods(), ["reactions.add"]);
+});
+
+test("recovered pin and unpin rows are no-ops when the message is already in the requested state", async () => {
+ const pinned = harness({ pinned: ["100.200"] });
+ await pinned.deliver({ destination: { pin: { messageTs: "100.200" } }, ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(pinned.methods(), ["pins.list"]);
+ assert.deepEqual(pinned.acknowledgements, ["D1"]);
+
+ const unpinned = harness({ pinned: [] });
+ await unpinned.deliver({ destination: { pin: { messageTs: "100.200", remove: true } }, ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(unpinned.methods(), ["pins.list"]);
+});
+
+test("recovered pin and unpin rows act when Slack disagrees with the requested state", async () => {
+ const pin = harness({ pinned: [] });
+ await pin.deliver({ destination: { pin: { messageTs: "100.200" } }, ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(pin.methods(), ["pins.list", "pins.add"]);
+
+ const unpin = harness({ pinned: ["100.200"] });
+ await unpin.deliver({ destination: { pin: { messageTs: "100.200", remove: true } }, ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(unpin.methods(), ["pins.list", "pins.remove"]);
+});
+
+test("a recovered delete row is a no-op once the message is gone", async () => {
+ const gone = harness({ thread: [] });
+ await gone.deliver({ destination: { delete: { messageTs: "100.500" } }, ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(gone.methods(), ["conversations.replies"]);
+ assert.deepEqual(gone.acknowledgements, ["D1"]);
+
+ const missingThread = harness({ repliesError: "thread_not_found" });
+ await missingThread.deliver({ destination: { delete: { messageTs: "100.500" } }, ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(missingThread.methods(), ["conversations.replies"]);
+});
+
+test("a recovered delete row deletes when the message still exists, and a fresh row deletes without probing", async () => {
+ const present = harness({ thread: [{ ts: "100.500", text: "to remove" }] });
+ await present.deliver({ destination: { delete: { messageTs: "100.500" } }, ageMs: RECOVERED_AGE_MS });
+ assert.deepEqual(present.methods(), ["conversations.replies", "chat.delete"]);
+ assert.equal(present.calls[0]!.args.ts, "100.500");
+
+ const fresh = harness({ thread: [] });
+ await fresh.deliver({ destination: { delete: { messageTs: "100.500" } } });
+ assert.deepEqual(fresh.methods(), ["chat.delete"]);
+});
diff --git a/test/slack-unprompted-ack.test.ts b/test/slack-unprompted-ack.test.ts
index 0118dd617..f4cced04b 100644
--- a/test/slack-unprompted-ack.test.ts
+++ b/test/slack-unprompted-ack.test.ts
@@ -14,6 +14,7 @@ const directory = {
function harness(run: (hooks: any) => Promise<unknown>) {
const reactions: string[] = [];
const posts: string[] = [];
+ const postArgs: Array<Record<string, unknown>> = [];
const turns: any[] = [];
const client = {
reactions: {
@@ -25,14 +26,21 @@ function harness(run: (hooks: any) => Promise<unknown>) {
},
},
chat: {
- postMessage: async ({ text }: { text: string }) => {
- posts.push(text);
+ postMessage: async (args: Record<string, unknown>) => {
+ posts.push(args.text as string);
+ postArgs.push(args);
return { ts: `9${posts.length}.0` };
},
+ update: async () => ({}),
+ delete: async () => ({}),
},
};
const handler = createTurnHandler({
- core: { stageBlob: async () => ({ blobId: "b", sizeBytes: 0 }) },
+ core: {
+ stageBlob: async () => ({ blobId: "b", sizeBytes: 0 }),
+ reportRunEditRef: async () => {},
+ reportTurnMetrics: async () => {},
+ },
flow: {
inFlightRuns: { add() {}, delete() {}, has: () => false },
inFlightRunByThread: { get() {}, set() {}, clear() {} },
@@ -74,9 +82,23 @@ function harness(run: (hooks: any) => Promise<unknown>) {
unprompted: true,
prefetched: { actor, info: { id: "C1", name: "eng" } as any, audience: [actor] },
};
- return { handler, client, followup, reactions, posts, turns };
+ return { handler, client, followup, reactions, posts, postArgs, turns };
}
+test("the task-list placeholder is posted with its run's status key so a recovered reply can find it", async () => {
+ const h = harness(async (hooks) => {
+ await hooks.onQueued?.("r1");
+ await hooks.onTasks?.([{ id: "t1", title: "Check staging", status: "in_progress" }]);
+ return { status: "ok", reply: "Staging is green." };
+ });
+ await h.handler.handleIncoming({ ...h.followup, unprompted: false, rawText: "<@UBOT> check staging" }, h.client);
+ const placeholder = h.postArgs.find((args) => String(args.text).includes("Check staging"));
+ assert.deepEqual(placeholder?.metadata, {
+ event_type: "qm_delivery",
+ event_payload: { idempotency_key: "status:run:r1" },
+ });
+});
+
test("an unmentioned thread followup QM answers gets the ack reaction added then cleared", async () => {
const h = harness(async (hooks) => {
hooks.onReplying?.();