Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Wait for attachment delivery in live Slack qualification (#1468)

Regan Bell6b38e4a2fcc5
diff --git a/test/live-slack-attachment-wait.test.ts b/test/live-slack-attachment-wait.test.ts
new file mode 100644
index 000000000..c3c284d70
--- /dev/null
+++ b/test/live-slack-attachment-wait.test.ts
@@ -0,0 +1,114 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { setImmediate } from "node:timers/promises";
+import { Ctx, DmHandle, type Env } from "./live-slack/harness.ts";
+import { SlackClient, type SlackMessage } from "./live-slack/slack.ts";
+import { scenarios } from "./live-slack/scenarios.ts";
+
+function dmWithHistory(history: () => Promise<SlackMessage[]>): DmHandle {
+ const qa = new SlackClient("synthetic-token", "http://127.0.0.1");
+ qa.history = history;
+ qa.getPermalink = async () => "https://example.test/message";
+ const env = { qa, botUserId: "BOT" } as Env;
+ return new DmHandle(new Ctx(env, { name: "attachment", lane: "dm", run: async () => {} }, 1), "DM");
+}
+
+const acknowledgement = { ts: "2", user: "BOT", text: "I'll create that file." };
+const hasRequestedFile = (message: SlackMessage) =>
+ message.files?.some((file) => file.name === "requested.txt") === true;
+
+test("file waits ignore stable acknowledgements and unrelated files until the requested upload arrives", async (t) => {
+ t.mock.timers.enable({ apis: ["Date", "setTimeout"], now: 100_000 });
+ const file: SlackMessage = { ts: "4", user: "BOT", files: [{ name: "requested.txt" }] };
+ const dm = dmWithHistory(async () => [
+ acknowledgement,
+ { ts: "3", user: "BOT", files: [{ name: "unrelated.txt" }] },
+ ...(Date.now() >= 110_000 ? [file] : []),
+ ]);
+ let settled = false;
+ const result = dm.waitForBotReply("1", { timeoutMs: 20_000, accept: hasRequestedFile }).then((message) => {
+ settled = true;
+ return message;
+ });
+ await setImmediate();
+ for (let i = 0; i < 3; i++) {
+ t.mock.timers.tick(2500);
+ await setImmediate();
+ }
+ assert.equal(settled, false);
+ for (let i = 0; i < 4; i++) {
+ t.mock.timers.tick(2500);
+ await setImmediate();
+ }
+ assert.deepEqual(await result, file);
+});
+
+test("file waits fail at the deadline when only acknowledgements or another user's file arrive", async (t) => {
+ t.mock.timers.enable({ apis: ["Date", "setTimeout"], now: 100_000 });
+ const dm = dmWithHistory(async () => [
+ acknowledgement,
+ { ts: "3", user: "OTHER", files: [{ name: "requested.txt" }] },
+ ]);
+ const rejected = assert.rejects(
+ dm.waitForBotReply("1", { timeoutMs: 10_000, accept: hasRequestedFile }),
+ /timed out after 10000ms/,
+ );
+ await setImmediate();
+ for (let i = 0; i < 4; i++) {
+ t.mock.timers.tick(2500);
+ await setImmediate();
+ }
+ await rejected;
+});
+
+test("ordinary reply waits retain their existing stable-text behavior", async (t) => {
+ t.mock.timers.enable({ apis: ["Date", "setTimeout"], now: 100_000 });
+ const dm = dmWithHistory(async () => [acknowledgement]);
+ const result = dm.waitForBotReply("1", { timeoutMs: 10_000 });
+ await setImmediate();
+ for (let i = 0; i < 2; i++) {
+ t.mock.timers.tick(2500);
+ await setImmediate();
+ }
+ assert.deepEqual(await result, acknowledgement);
+});
+
+for (const evidence of ["none", "attachment", "slack-link", "markdown-link", "url"] as const) {
+ test(`channel file scenario requires delivery evidence: ${evidence}`, async (t) => {
+ t.mock.timers.enable({ apis: ["Date", "setTimeout"], now: 100_000 });
+ const scenario = scenarios.find((item) => item.name === "file-upload")!;
+ const qa = new SlackClient("synthetic-token", "http://127.0.0.1");
+ qa.createChannel = async () => "CHANNEL";
+ qa.invite = async () => {};
+ qa.post = async () => "1";
+ qa.getPermalink = async () => "https://example.test/message";
+ const ctx = new Ctx({ qa, botUserId: "BOT", runId: "attachment" } as Env, scenario, 1);
+ const filename = `${ctx.marker()}.txt`;
+ const payload: Partial<SlackMessage> = {};
+ if (evidence === "attachment") payload.files = [{ name: filename }];
+ else if (evidence === "slack-link") payload.text = `<https://example.test/download|${filename}>`;
+ else if (evidence === "markdown-link") payload.text = `[${filename}](https://example.test/download)`;
+ else payload.text = `https://example.test/${filename}`;
+ qa.replies = async () => [
+ { ts: "2", user: "BOT", text: `I'll create ${filename} now.` },
+ ...(evidence !== "none" && Date.now() >= 110_000 ? [{ ts: "3", user: "BOT", ...payload }] : []),
+ ];
+ let settled = false;
+ const run = scenario.run(ctx).then(() => {
+ settled = true;
+ });
+ const result = evidence === "none" ? assert.rejects(run, /timed out/) : run;
+ await setImmediate();
+ for (let i = 0; i < 3; i++) {
+ t.mock.timers.tick(2500);
+ await setImmediate();
+ }
+ assert.equal(settled, false);
+ for (let i = 0; i < 130; i++) {
+ t.mock.timers.tick(2500);
+ await setImmediate();
+ }
+ await result;
+ assert.equal(settled, evidence !== "none");
+ });
+}
diff --git a/test/live-slack/harness.ts b/test/live-slack/harness.ts
index 590b1e916..d38ad9763 100644
--- a/test/live-slack/harness.ts
+++ b/test/live-slack/harness.ts
@@ -81,6 +81,7 @@ const STABLE_MS = 5000;
export interface WaitOpts {
timeoutMs?: number;
match?: RegExp;
+ accept?: (message: SlackMessage) => boolean;
afterTs?: string;
onFrame?: (text: string) => void;
record?: (msgTs: string, text: string) => void;
@@ -115,6 +116,7 @@ async function waitForFinalBotMessage(
const text = m.text ?? "";
if (isLiveStatusText(text)) continue;
if (opts.match && !opts.match.test(text)) continue;
+ if (opts.accept && !opts.accept(m)) continue;
const entry = seen.get(m.ts);
if (entry && entry.text === text && Date.now() - entry.firstSeen >= STABLE_MS) return m;
}
diff --git a/test/live-slack/scenarios.ts b/test/live-slack/scenarios.ts
index a84842d3a..9adbb308a 100644
--- a/test/live-slack/scenarios.ts
+++ b/test/live-slack/scenarios.ts
@@ -1,7 +1,7 @@
import { readFileSync } from "node:fs";
import path from "node:path";
import { assert, isLiveStatusText, type Scenario } from "./harness.ts";
-import { sleep } from "./slack.ts";
+import { sleep, type SlackMessage } from "./slack.ts";
import { assertRuntimeHandoff } from "./runtime-handoff.ts";
import { multiUserScenarios } from "./scenarios-multiuser.ts";
import { twinScenarios } from "./scenarios-twin.ts";
@@ -96,7 +96,12 @@ export const scenarios: Scenario[] = [
});
await ch.mention("fwd this screenshot to my DM with you, let's send a fix", fileTs);
await ch.waitForBotReply(fileTs, { timeoutMs: SANDBOX_TIMEOUT - 30_000 });
- const dmMsg = await dm.waitForBotReply(baseline, { timeoutMs: 90_000 }).catch(() => null);
+ const dmMsg = await dm
+ .waitForBotReply(baseline, {
+ timeoutMs: 90_000,
+ accept: (message) => (message.files ?? []).some((file) => (file.name ?? file.title ?? "").includes(marker)),
+ })
+ .catch(() => null);
const forwarded = !!dmMsg && (dmMsg.files ?? []).some((f) => (f.name ?? f.title ?? "").includes(marker));
assert.ok(
forwarded,
@@ -220,13 +225,19 @@ export const scenarios: Scenario[] = [
const root = await ch.mention(
`Create a text file named ${marker}.txt containing the single line "hello from ci" and share the file here in this thread.`,
);
- await ch.waitForBotReply(root, { timeoutMs: SANDBOX_TIMEOUT - 30_000 });
+ const hasSharedFile = (message: SlackMessage) =>
+ (message.files ?? []).some((file) => (file.name ?? file.title ?? "").includes(marker)) ||
+ [
+ ...(message.text ?? "").matchAll(
+ /<https?:\/\/[^>\s|]+(?:\|[^>]+)?>|\[[^\]]+\]\(https?:\/\/[^)\s]+\)|https?:\/\/[^\s<>]+/g,
+ ),
+ ].some(([link]) => link.includes(`${marker}.txt`));
+ await ch.waitForBotReply(root, {
+ timeoutMs: SANDBOX_TIMEOUT - 30_000,
+ accept: hasSharedFile,
+ });
const botMsgs = await ch.botMessagesInThread(root);
- const shared = botMsgs.some(
- (m) =>
- (m.files ?? []).some((f) => (f.name ?? f.title ?? "").includes(marker)) ||
- (m.text ?? "").includes(`${marker}.txt`),
- );
+ const shared = botMsgs.some(hasSharedFile);
assert.ok(shared, `no shared file or file link mentioning ${marker}.txt found in thread`);
},
},
@@ -397,7 +408,10 @@ export const scenarios: Scenario[] = [
const ts = await dm.send(
`Create a text file named ${marker}.txt containing the single line "hello from ci" and send me the file here.`,
);
- await dm.waitForBotReply(ts, { timeoutMs: SANDBOX_TIMEOUT - 30_000 });
+ await dm.waitForBotReply(ts, {
+ timeoutMs: SANDBOX_TIMEOUT - 30_000,
+ accept: (message) => (message.files ?? []).some((file) => (file.name ?? file.title ?? "").includes(marker)),
+ });
const msgs = await ctx.env.qa.history(dm.id, ts);
const delivered = msgs.some(
(m) =>