Multiplayer agent harness for work.
Backfill Slack history once when the mirror starts late (#1695)
Joshua France3cb05a2040d2
diff --git a/src/api/slack-core-client.ts b/src/api/slack-core-client.tsindex 8b3d2adf8..f94d4b441 100644--- a/src/api/slack-core-client.ts+++ b/src/api/slack-core-client.ts@@ -30,6 +30,7 @@ import type {TurnResult,} from "../types.ts";import { scopeId } from "../types.ts";+import type { MemoryService } from "../memory/memory-service.ts";import type { CachedMessage, ReadMessagesOpts, SurfaceCache, IngestEvent } from "../surface-cache/surface-cache.ts";import type { AckEmojiPickStore } from "../surface-cache/ack-emoji-pick-store.ts";import type { OrgBranding, ScopedConfigStore } from "../resolution/config-store.ts";@@ -116,6 +117,7 @@ export interface SlackCoreClient {readBlob(blobId: string): Promise<Buffer>;readFileArtifact(artifactId: string, viewerId: string): Promise<Buffer>;rememberSurfaceHistory?(events: IngestEvent[]): Promise<void>;+ noteSurfaceHistoryGap?(container: string, note: string): Promise<void>;readSurfaceMessages?(container: string, opts?: ReadMessagesOpts): Promise<CachedMessage[]>;ingestSurfaceEvents(events: IngestEvent[], self?: { name?: string; mentionId?: string }): Promise<void>;submitTurn(body: Omit<TurnRequest, "surface">): Promise<TurnResult>;@@ -195,6 +197,7 @@ export interface SlackCoreClientDeps {leaderLease?: LeaderLease;stagedEnvelopes?: DurableMap<StagedEnvelope>;surfaceCache?: SurfaceCache;+ memory?: MemoryService;inboxEvent?(event: ConversationEvent): Promise<void>;}@@ -332,6 +335,12 @@ export function createSlackCoreClient(deps: SlackCoreClientDeps): SlackCoreClienawait deps.surfaceCache?.ingest(events);},+ async noteSurfaceHistoryGap(container, note) {+ if (!deps.memory) return;+ const kind = (await deps.surfaceCache?.containerState(container))?.kind;+ await deps.memory.capture(scopeId(kind === "group" ? "group" : "channel", container), [note], Date.now());+ },+async readSurfaceMessages(container, opts) {return deps.app.readSurfaceMessages(container, { ...opts, noFallback: true });},diff --git a/src/core/orchestrator/surface-tools.ts b/src/core/orchestrator/surface-tools.tsindex 74537458c..c04a3d1d8 100644--- a/src/core/orchestrator/surface-tools.ts+++ b/src/core/orchestrator/surface-tools.ts@@ -378,8 +378,9 @@ export function createSurfaceToolDeps(ctx: SurfaceToolsContext): SurfaceToolDepsreturn {ok: false,message:- "[live search runs as the asking person's own Slack login, and this turn has no connected one — " +- "if a person asked, point them at " ++ "[live search runs as the asking person's own Slack login, and this turn has no connected one. " ++ "Reading this conversation's history needs no personal login, so read it before concluding anything is missing; " ++ "only if the message is elsewhere, and a person asked, point them at " +(deps.publicWebUrl? `${deps.publicWebUrl.replace(/\/$/, "")}/connect/slack/self-connect`: "the web UI's Connectors page") +diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.tsindex 535c9d246..27b21f75f 100644--- a/src/harness/agent-tools.ts+++ b/src/harness/agent-tools.ts@@ -3347,8 +3347,8 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):"and only back to each channel's coverage date, so it can miss both very recent and older messages. " +"`slack` runs Slack's own full-history search AS THE ASKING PERSON, via their connected Slack login — it sees " +"exactly what they can see, including their own DMs and private channels (never anyone else's). Retry with " +- "`slack` whenever the mirror comes up empty; if the asker hasn't connected Slack, the result says where they " +- "can connect it themselves. A message neither lens can see may still exist — say what you couldn't search, don't declare it nonexistent. " ++ "`slack` when the mirror and this conversation's own history both come up empty; reading the conversation needs no personal login. " ++ "A message neither lens can see may still exist — say what you couldn't search, don't declare it nonexistent. " +"In a channel or group, hits from the asker's DMs or private channels are for their eyes: don't quote that content to the room — acknowledge you found it and take it to their DM.",}),),diff --git a/src/slack/events.ts b/src/slack/events.tsindex 11a39f77d..a9a83a60b 100644--- a/src/slack/events.ts+++ b/src/slack/events.ts@@ -48,6 +48,7 @@ export function registerSlackEvents(allowActor?: (actor: ActorAssertion) => boolean;denyResponder?: DenyResponder;webUiPublicUrl?: string;+ backfillHistory?: (client: unknown, channel: string) => Promise<unknown>;ensureHeader?: (client: SurfaceHeaderClient,channel: string,@@ -356,6 +357,7 @@ export function registerSlackEvents(const e = parseLifecycleEvent(event);if (deduper.seen(dedupeKey({ event_id: parseEventId(body), channel: e.channel, ts: e.eventTs }))) return;if (e.user === ids.botUserId) {+ if (e.channel && deps.backfillHistory) await deps.backfillHistory(client, e.channel).catch(() => {});if (allowActor) {await forceDirectorySync(client);return;diff --git a/src/slack/history.ts b/src/slack/history.tsindex 1510d595e..d3cdf75c0 100644--- a/src/slack/history.ts+++ b/src/slack/history.ts@@ -202,6 +202,18 @@ export function createSlackHistoryReader(deps: {}}+ async function remember(channel: string, messages: SlackHistoryMessage[]): Promise<void> {+ if (!messages.length || !deps.core.rememberSurfaceHistory) return;+ await deps.core+ .rememberSurfaceHistory(+ messages+ .filter((m) => m.ts)+ .map((m) => slackMessageToIngestEvent({ ...m, channel }, deps.ids, { handled: true })),+ )+ .catch((error) => swallow("slack: history mirror ingest", error));+ }++ const backfilled = new Set<string>();let shadowPending = false;return async (client, channel, threadTs, before, expandThreads) => {if (deps.source !== "mirror") {@@ -215,7 +227,9 @@ export function createSlackHistoryReader(deps: {return live;}const historyClient = deps.historyClient ?? client;+ const key = threadTs ? `${channel}:${threadTs}` : channel;let mirrored: SlackHistoryMessage[] = [];+ let mirrorResult: SlackHistoryPage | undefined;const deleted = new Set<string>();if (deps.core.readSurfaceMessages) {try {@@ -245,7 +259,7 @@ export function createSlackHistoryReader(deps: {: {}),}));if (mirrored.length && (!threadTs || rows.some((m) => m.ts === threadTs) || before)) {- return {+ mirrorResult = {raw: mirrored,hasMore: mirrorPage.hasMore,...(mirrorPage.truncatedExpansions !== undefined@@ -253,28 +267,28 @@ export function createSlackHistoryReader(deps: {: {}),note: MIRROR_CONTEXT_NOTE,};+ if (before) return mirrorResult;}} catch (error) {swallow("slack: mirror context read", error);}}+ if (!before && backfilled.has(key))+ return mirrorResult ?? { raw: mirrored, hasMore: true, note: MIRROR_CONTEXT_NOTE };try {- const paging = { channel, limit, ...(before ? { latest: before, inclusive: false } : {}) };+ if (!before) backfilled.add(key);+ const paging = {+ channel,+ limit: mirrorResult ? Math.min(limit, SHARED_SLACK_HISTORY_LIMIT) : limit,+ ...(before ? { latest: before, inclusive: false } : {}),+ };const page = parseMessageList(threadTs? await historyClient.conversations.replies({ ...paging, ts: threadTs }): await historyClient.conversations.history(paging),);page.messages = page.messages.filter((m) => !isOwnStatusCard(m, deps.ids.botUserId, deps.ids.ownBotId));- if (page.messages.length && deps.core.rememberSurfaceHistory) {- await deps.core- .rememberSurfaceHistory(- page.messages- .filter((m) => m.ts)- .map((m) => slackMessageToIngestEvent({ ...m, channel }, deps.ids, { handled: true })),- )- .catch((error) => swallow("slack: history mirror ingest", error));- }+ await remember(channel, page.messages);if (deps.core.readSurfaceMessages && page.messages.length) {const stored = await deps.core.readSurfaceMessages(channel, {timestamps: page.messages.flatMap((m) => (m.ts ? [m.ts] : [])),@@ -293,8 +307,20 @@ export function createSlackHistoryReader(deps: {...(page.hasMore ? { note: "Slack history is truncated; older messages may be absent." } : {}),};} catch (error) {+ if (!before && slackHistoryRateLimitMessage(error) && deps.core.noteSurfaceHistoryGap)+ await deps.core+ .noteSurfaceHistoryGap(+ channel,+ `Slack rate-limited the initial history pull for ${threadTs ? `thread ${threadTs} in ` : ""}this channel, so messages from before QM joined were not loaded.`,+ )+ .catch((noteError) => swallow("slack: history gap memory note", noteError));if (!mirrored.length) throw error;swallow("slack: incomplete mirror history fallback", error);+ if (mirrorResult)+ return {+ ...mirrorResult,+ note: [MIRROR_CONTEXT_NOTE, slackHistoryRateLimitMessage(error, deps)].filter(Boolean).join(" "),+ };return {raw: mirrored,hasMore: true,diff --git a/src/slack/index.ts b/src/slack/index.tsindex feca1cd14..88df5e36c 100644--- a/src/slack/index.ts+++ b/src/slack/index.ts@@ -416,6 +416,9 @@ export async function startSlackPlugin(...(allowActor ? { allowActor } : {}),...(denyResponder ? { denyResponder } : {}),...(cfg.webUiPublicUrl ? { webUiPublicUrl: cfg.webUiPublicUrl } : {}),+ ...((cfg.contextSource ?? "live") === "mirror"+ ? { backfillHistory: (client: unknown, channel: string) => readHistory(client, channel) }+ : {}),ensureHeader,});const surfaceContext = createSurfaceContextFulfiller({diff --git a/src/wiring.ts b/src/wiring.tsindex 393e0c28e..7ed8a86e3 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -2182,6 +2182,7 @@ export function buildApp(});const slackCore = createSlackCoreClient({identity,+ memory,...(keychain? {keychainApprovals: createKeychainApprovals({diff --git a/test/slack-mirror-backfill.test.ts b/test/slack-mirror-backfill.test.tsnew file mode 100644index 000000000..3c310bc6f--- /dev/null+++ b/test/slack-mirror-backfill.test.ts@@ -0,0 +1,183 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { createMemorySurfaceCache } from "../src/surface-cache/surface-cache.ts";+import { createSlackHistoryReader } from "../src/slack/history.ts";+import { createSlackCoreClient, type SlackCoreClient } from "../src/api/slack-core-client.ts";+import { createTurnStream } from "../src/runs/turn-stream.ts";+import type { BotIdentity } from "../src/slack/directory.ts";+import { registerSlackEvents } from "../src/slack/events.ts";+import { createDeduper } from "../src/slack/lib.ts";++const ids = { botUserId: "UBOT", ownBotId: "BBOT" } as BotIdentity;++function fixture(fail?: Error) {+ const cache = createMemorySurfaceCache();+ const notes: string[] = [];+ const core = {+ noteSurfaceHistoryGap: async (container: string, note: string) => {+ notes.push(`${container}:${note}`);+ },+ readSurfaceMessages: cache.readMessages,+ rememberSurfaceHistory: async (events) => {+ await cache.ingest(events);+ },+ } as SlackCoreClient;+ const calls: string[] = [];+ const historyClient = {+ conversations: {+ history: async (args: any) => {+ calls.push(`history:${args.latest ?? ""}:${args.limit}`);+ if (fail) throw fail;+ return {+ messages: [+ { ts: "20.000000", text: "root", user: "U1", thread_ts: "20.000000", reply_count: 1 },+ { ts: "10.000000", text: "earlier plan", user: "U2" },+ ],+ };+ },+ replies: async (args: any) => {+ calls.push(`replies:${args.ts}:${args.limit}`);+ if (fail) throw fail;+ return {+ messages: [+ { ts: "20.000000", text: "root", user: "U1", thread_ts: "20.000000" },+ { ts: "21.000000", text: "what did we decide above?", user: "U1", thread_ts: "20.000000" },+ ],+ };+ },+ },+ };+ const readHistory = createSlackHistoryReader({ core, ids, source: "mirror", historyClient });+ return { cache, calls, notes, readHistory };+}++async function joinedLate(cache: ReturnType<typeof createMemorySurfaceCache>) {+ await cache.ingest([+ { container: "C1", ts: "20.000000", text: "root", authorId: "U1" },+ { container: "C1", ts: "21.000000", sub: "20.000000", text: "what did we decide above?", authorId: "U1" },+ ]);+}++test("a late-joined thread backfills its last 15 replies once", async () => {+ const { cache, calls, readHistory } = fixture();+ await joinedLate(cache);+ const thread = await readHistory({}, "C1", "20.000000", undefined, true);+ assert.deepEqual(thread.raw.map((m) => m.ts).sort(), ["20.000000", "21.000000"]);+ assert.deepEqual(calls, ["replies:20.000000:15"]);+ await readHistory({}, "C1", "20.000000", undefined, true);+ assert.equal(calls.length, 1);+ const channel = await readHistory({}, "C1");+ assert.ok(channel.raw.some((m) => m.text === "earlier plan"));+ await readHistory({}, "C1");+ assert.deepEqual(calls, ["replies:20.000000:15", "history::15"]);+});++test("a failed backfill is not retried and falls back to the mirror", async () => {+ const limited = Object.assign(new Error("rate limited"), {+ code: "slack_webapi_rate_limited_error",+ retryAfter: 30,+ });+ const { cache, calls, readHistory } = fixture(limited);+ await joinedLate(cache);+ const first = await readHistory({}, "C1", "20.000000", undefined, true);+ assert.deepEqual(first.raw.map((m) => m.ts).sort(), ["20.000000", "21.000000"]);+ assert.deepEqual(calls, ["replies:20.000000:15"]);+ const again = await readHistory({}, "C1", "20.000000", undefined, true);+ assert.deepEqual(again.raw.map((m) => m.ts).sort(), ["20.000000", "21.000000"]);+ assert.equal(calls.length, 1);+});++test("a thread read before its root is mirrored still fetches only once", async () => {+ const { cache, calls, readHistory } = fixture(new Error("unavailable"));+ await cache.ingest([+ { container: "C1", ts: "21.000000", sub: "20.000000", text: "what did we decide above?", authorId: "U1" },+ ]);+ await readHistory({}, "C1", "20.000000");+ const again = await readHistory({}, "C1", "20.000000");+ assert.deepEqual(+ again.raw.map((m) => m.ts),+ ["21.000000"],+ );+ assert.deepEqual(calls, ["replies:20.000000:200"]);+});++test("an older-page read does not use up the latest-page backfill", async () => {+ const { cache, calls, readHistory } = fixture();+ await joinedLate(cache);+ await readHistory({}, "C1", undefined, "15.000000");+ await readHistory({}, "C1");+ assert.deepEqual(calls, ["history:15.000000:200", "history::15"]);+});++test("a rate-limited backfill notes the gap in the channel's memory once", async () => {+ const limited = Object.assign(new Error("rate limited"), { code: "slack_webapi_rate_limited_error" });+ const { cache, notes, readHistory } = fixture(limited);+ await joinedLate(cache);+ await readHistory({}, "C1", "20.000000", undefined, true);+ await readHistory({}, "C1", "20.000000", undefined, true);+ assert.equal(notes.length, 1);+ assert.match(notes[0]!, /^C1:Slack rate-limited the initial history pull/);+});++test("a non-rate-limit backfill failure writes no memory note", async () => {+ const { cache, notes, readHistory } = fixture(new Error("boom"));+ await joinedLate(cache);+ await readHistory({}, "C1", "20.000000", undefined, true);+ assert.deepEqual(notes, []);+});++test("the history gap note lands in the conversation's own scope memory", async () => {+ const cache = createMemorySurfaceCache();+ await cache.ingest([+ { container: "C1", ts: "1.000000", text: "hi", authorId: "U1", kind: "channel" },+ { container: "G1", ts: "1.000000", text: "hi", authorId: "U1", kind: "group" },+ ]);+ const captured: string[] = [];+ const client = createSlackCoreClient({+ surfaceCache: cache,+ memory: {+ capture: async (scope: string, facts: string[]) => {+ captured.push(`${scope}=${facts.join()}`);+ return facts.length;+ },+ },+ turnStream: createTurnStream(),+ runs: { onTerminal() {} },+ } as any);+ await client.noteSurfaceHistoryGap!("C1", "gap");+ await client.noteSurfaceHistoryGap!("G1", "gap");+ assert.deepEqual(captured, ["channel:C1=gap", "group:G1=gap"]);+});++test("the bot joining a channel backfills it once, and a failed pull never blocks the join", async () => {+ const events = new Map<string, (args: any) => Promise<void>>();+ const pulled: string[] = [];+ let synced = 0;+ registerSlackEvents({ event: (n: string, h: any) => void events.set(n, h), message: () => {} } as any, {+ handler: {} as any,+ mirror: {} as any,+ directory: {+ forceDirectorySync: async () => {+ synced++;+ },+ } as any,+ ids,+ deduper: createDeduper(),+ allowActor: () => true,+ backfillHistory: async (_client, channel) => {+ pulled.push(channel);+ if (channel === "C2") throw new Error("rate limited");+ },+ });+ const join = (channel: string, user: string) =>+ events.get("member_joined_channel")!({+ event: { channel, user, event_ts: `${channel}${user}` },+ body: {},+ client: {},+ });+ await join("C1", "UBOT");+ await join("C1", "U9");+ await join("C2", "UBOT");+ assert.deepEqual(pulled, ["C1", "C2"]);+ assert.equal(synced, 3);+});