Multiplayer agent harness for work.
Wait for attachment delivery in live Slack qualification (#1468)
Regan Bell6b38e4a2fcc5
diff --git a/test/live-slack-attachment-wait.test.ts b/test/live-slack-attachment-wait.test.tsnew file mode 100644index 000000000..c3c284d70--- /dev/null+++ b/test/live-slack-attachment-wait.test.ts@@ -0,0 +1,114 @@+import assert from "node:assert/strict";+import test from "node:test";+import { setImmediate } from "node:timers/promises";+import { Ctx, DmHandle, type Env } from "./live-slack/harness.ts";+import { SlackClient, type SlackMessage } from "./live-slack/slack.ts";+import { scenarios } from "./live-slack/scenarios.ts";++function dmWithHistory(history: () => Promise<SlackMessage[]>): DmHandle {+ const qa = new SlackClient("synthetic-token", "http://127.0.0.1");+ qa.history = history;+ qa.getPermalink = async () => "https://example.test/message";+ const env = { qa, botUserId: "BOT" } as Env;+ return new DmHandle(new Ctx(env, { name: "attachment", lane: "dm", run: async () => {} }, 1), "DM");+}++const acknowledgement = { ts: "2", user: "BOT", text: "I'll create that file." };+const hasRequestedFile = (message: SlackMessage) =>+ message.files?.some((file) => file.name === "requested.txt") === true;++test("file waits ignore stable acknowledgements and unrelated files until the requested upload arrives", async (t) => {+ t.mock.timers.enable({ apis: ["Date", "setTimeout"], now: 100_000 });+ const file: SlackMessage = { ts: "4", user: "BOT", files: [{ name: "requested.txt" }] };+ const dm = dmWithHistory(async () => [+ acknowledgement,+ { ts: "3", user: "BOT", files: [{ name: "unrelated.txt" }] },+ ...(Date.now() >= 110_000 ? [file] : []),+ ]);+ let settled = false;+ const result = dm.waitForBotReply("1", { timeoutMs: 20_000, accept: hasRequestedFile }).then((message) => {+ settled = true;+ return message;+ });+ await setImmediate();+ for (let i = 0; i < 3; i++) {+ t.mock.timers.tick(2500);+ await setImmediate();+ }+ assert.equal(settled, false);+ for (let i = 0; i < 4; i++) {+ t.mock.timers.tick(2500);+ await setImmediate();+ }+ assert.deepEqual(await result, file);+});++test("file waits fail at the deadline when only acknowledgements or another user's file arrive", async (t) => {+ t.mock.timers.enable({ apis: ["Date", "setTimeout"], now: 100_000 });+ const dm = dmWithHistory(async () => [+ acknowledgement,+ { ts: "3", user: "OTHER", files: [{ name: "requested.txt" }] },+ ]);+ const rejected = assert.rejects(+ dm.waitForBotReply("1", { timeoutMs: 10_000, accept: hasRequestedFile }),+ /timed out after 10000ms/,+ );+ await setImmediate();+ for (let i = 0; i < 4; i++) {+ t.mock.timers.tick(2500);+ await setImmediate();+ }+ await rejected;+});++test("ordinary reply waits retain their existing stable-text behavior", async (t) => {+ t.mock.timers.enable({ apis: ["Date", "setTimeout"], now: 100_000 });+ const dm = dmWithHistory(async () => [acknowledgement]);+ const result = dm.waitForBotReply("1", { timeoutMs: 10_000 });+ await setImmediate();+ for (let i = 0; i < 2; i++) {+ t.mock.timers.tick(2500);+ await setImmediate();+ }+ assert.deepEqual(await result, acknowledgement);+});++for (const evidence of ["none", "attachment", "slack-link", "markdown-link", "url"] as const) {+ test(`channel file scenario requires delivery evidence: ${evidence}`, async (t) => {+ t.mock.timers.enable({ apis: ["Date", "setTimeout"], now: 100_000 });+ const scenario = scenarios.find((item) => item.name === "file-upload")!;+ const qa = new SlackClient("synthetic-token", "http://127.0.0.1");+ qa.createChannel = async () => "CHANNEL";+ qa.invite = async () => {};+ qa.post = async () => "1";+ qa.getPermalink = async () => "https://example.test/message";+ const ctx = new Ctx({ qa, botUserId: "BOT", runId: "attachment" } as Env, scenario, 1);+ const filename = `${ctx.marker()}.txt`;+ const payload: Partial<SlackMessage> = {};+ if (evidence === "attachment") payload.files = [{ name: filename }];+ else if (evidence === "slack-link") payload.text = `<https://example.test/download|${filename}>`;+ else if (evidence === "markdown-link") payload.text = `[${filename}](https://example.test/download)`;+ else payload.text = `https://example.test/${filename}`;+ qa.replies = async () => [+ { ts: "2", user: "BOT", text: `I'll create ${filename} now.` },+ ...(evidence !== "none" && Date.now() >= 110_000 ? [{ ts: "3", user: "BOT", ...payload }] : []),+ ];+ let settled = false;+ const run = scenario.run(ctx).then(() => {+ settled = true;+ });+ const result = evidence === "none" ? assert.rejects(run, /timed out/) : run;+ await setImmediate();+ for (let i = 0; i < 3; i++) {+ t.mock.timers.tick(2500);+ await setImmediate();+ }+ assert.equal(settled, false);+ for (let i = 0; i < 130; i++) {+ t.mock.timers.tick(2500);+ await setImmediate();+ }+ await result;+ assert.equal(settled, evidence !== "none");+ });+}diff --git a/test/live-slack/harness.ts b/test/live-slack/harness.tsindex 590b1e916..d38ad9763 100644--- a/test/live-slack/harness.ts+++ b/test/live-slack/harness.ts@@ -81,6 +81,7 @@ const STABLE_MS = 5000;export interface WaitOpts {timeoutMs?: number;match?: RegExp;+ accept?: (message: SlackMessage) => boolean;afterTs?: string;onFrame?: (text: string) => void;record?: (msgTs: string, text: string) => void;@@ -115,6 +116,7 @@ async function waitForFinalBotMessage(const text = m.text ?? "";if (isLiveStatusText(text)) continue;if (opts.match && !opts.match.test(text)) continue;+ if (opts.accept && !opts.accept(m)) continue;const entry = seen.get(m.ts);if (entry && entry.text === text && Date.now() - entry.firstSeen >= STABLE_MS) return m;}diff --git a/test/live-slack/scenarios.ts b/test/live-slack/scenarios.tsindex a84842d3a..9adbb308a 100644--- a/test/live-slack/scenarios.ts+++ b/test/live-slack/scenarios.ts@@ -1,7 +1,7 @@import { readFileSync } from "node:fs";import path from "node:path";import { assert, isLiveStatusText, type Scenario } from "./harness.ts";-import { sleep } from "./slack.ts";+import { sleep, type SlackMessage } from "./slack.ts";import { assertRuntimeHandoff } from "./runtime-handoff.ts";import { multiUserScenarios } from "./scenarios-multiuser.ts";import { twinScenarios } from "./scenarios-twin.ts";@@ -96,7 +96,12 @@ export const scenarios: Scenario[] = [});await ch.mention("fwd this screenshot to my DM with you, let's send a fix", fileTs);await ch.waitForBotReply(fileTs, { timeoutMs: SANDBOX_TIMEOUT - 30_000 });- const dmMsg = await dm.waitForBotReply(baseline, { timeoutMs: 90_000 }).catch(() => null);+ const dmMsg = await dm+ .waitForBotReply(baseline, {+ timeoutMs: 90_000,+ accept: (message) => (message.files ?? []).some((file) => (file.name ?? file.title ?? "").includes(marker)),+ })+ .catch(() => null);const forwarded = !!dmMsg && (dmMsg.files ?? []).some((f) => (f.name ?? f.title ?? "").includes(marker));assert.ok(forwarded,@@ -220,13 +225,19 @@ export const scenarios: Scenario[] = [const root = await ch.mention(`Create a text file named ${marker}.txt containing the single line "hello from ci" and share the file here in this thread.`,);- await ch.waitForBotReply(root, { timeoutMs: SANDBOX_TIMEOUT - 30_000 });+ const hasSharedFile = (message: SlackMessage) =>+ (message.files ?? []).some((file) => (file.name ?? file.title ?? "").includes(marker)) ||+ [+ ...(message.text ?? "").matchAll(+ /<https?:\/\/[^>\s|]+(?:\|[^>]+)?>|\[[^\]]+\]\(https?:\/\/[^)\s]+\)|https?:\/\/[^\s<>]+/g,+ ),+ ].some(([link]) => link.includes(`${marker}.txt`));+ await ch.waitForBotReply(root, {+ timeoutMs: SANDBOX_TIMEOUT - 30_000,+ accept: hasSharedFile,+ });const botMsgs = await ch.botMessagesInThread(root);- const shared = botMsgs.some(- (m) =>- (m.files ?? []).some((f) => (f.name ?? f.title ?? "").includes(marker)) ||- (m.text ?? "").includes(`${marker}.txt`),- );+ const shared = botMsgs.some(hasSharedFile);assert.ok(shared, `no shared file or file link mentioning ${marker}.txt found in thread`);},},@@ -397,7 +408,10 @@ export const scenarios: Scenario[] = [const ts = await dm.send(`Create a text file named ${marker}.txt containing the single line "hello from ci" and send me the file here.`,);- await dm.waitForBotReply(ts, { timeoutMs: SANDBOX_TIMEOUT - 30_000 });+ await dm.waitForBotReply(ts, {+ timeoutMs: SANDBOX_TIMEOUT - 30_000,+ accept: (message) => (message.files ?? []).some((file) => (file.name ?? file.title ?? "").includes(marker)),+ });const msgs = await ctx.env.qa.history(dm.id, ts);const delivered = msgs.some((m) =>