Multiplayer agent harness for work.
Wait for channel membership before sandbox qualification (#1165)
Regan Bell3e5dc7a82027
diff --git a/src/api/routes/directory.ts b/src/api/routes/directory.tsindex 5a113731e..c1586ab33 100644--- a/src/api/routes/directory.ts+++ b/src/api/routes/directory.ts@@ -147,10 +147,22 @@ async function resolveDirectory(ctx: ApiCtx): Promise<void> {return sendJson(res, 200, { matches });}+async function channelMembership(ctx: ApiCtx): Promise<void> {+ return sendJson(ctx.res, 200, {+ member: await ctx.app.channelMember(ctx.params.channelId!, ctx.params.principalId!),+ });+}+export const directoryRoutes: ReadonlyArray<Route<ApiCtx>> = [{ method: "POST", path: "/v1/principals/:id/deactivate", auth: "source", handle: deactivatePrincipal },{ method: "POST", path: "/v1/principals/:id/reactivate", auth: "source", handle: reactivatePrincipal },{ method: "POST", path: "/v1/directory", auth: "source", handle: pushDirectory },{ method: "GET", path: "/v1/directory/meta", auth: "source", handle: directoryMeta },+ {+ method: "GET",+ path: "/v1/directory/channels/:channelId/members/:principalId",+ auth: "source",+ handle: channelMembership,+ },{ method: "GET", path: "/v1/directory/resolve", auth: "either", handle: resolveDirectory },];diff --git a/test/directory-resolve.test.ts b/test/directory-resolve.test.tsindex 6b5087815..bf4828a11 100644--- a/test/directory-resolve.test.ts+++ b/test/directory-resolve.test.ts@@ -6,6 +6,7 @@ import { mkdtempSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { buildApp, type BuiltApp } from "../src/wiring.ts";+import { signedRequestHeaders } from "../src/auth/source-auth-sign.ts";import { signRequest } from "../src/auth/source-auth.ts";import { createServer } from "../src/api/server.ts";import { mintCapabilityToken, CAPABILITY_TTL_MS } from "../src/auth/capability-token.ts";@@ -42,6 +43,25 @@ describe("GET /v1/directory/resolve (agent looks up a teammate's mention id)", aconst get = (path: string) => fetch(`${base}${path}`, { headers: { "x-agent-capability": cap } });+ it("membership reads require source authentication and report only persisted channel members", async () => {+ await built.app.upsertChannels(+ [{ channelId: "CPUBLIC", name: "public", isPrivate: false }],+ [{ channelId: "CPUBLIC", principalId: "[email protected]" }],+ );+ for (const [channel, principal, member] of [+ ["CPUBLIC", "[email protected]", true],+ ["CPUBLIC", "[email protected]", false],+ ["CUNKNOWN", "[email protected]", false],+ ] as const) {+ const path = `/v1/directory/channels/${channel}/members/${encodeURIComponent(principal)}`;+ assert.equal((await fetch(`${base}${path}`)).status, 401);+ assert.equal((await get(path)).status, 403);+ const response = await fetch(`${base}${path}`, { headers: signedRequestHeaders(SECRET, "GET", path) });+ assert.equal(response.status, 200);+ assert.deepEqual(await response.json(), { member });+ }+ });+it("resolves a name to a single match carrying the slackId needed to @-mention", async () => {const res = await get("/v1/directory/resolve?q=carol");assert.equal(res.status, 200);diff --git a/test/live-slack-request-bounds.test.ts b/test/live-slack-request-bounds.test.tsindex aba337c4d..962c9861c 100644--- a/test/live-slack-request-bounds.test.ts+++ b/test/live-slack-request-bounds.test.ts@@ -42,3 +42,90 @@ test("Core qualification request deadline aborts a stalled provider request", asawait new Promise<void>((resolve) => server.close(() => resolve()));}});++async function withMembershipServer(+ response: (path: string) => { status?: number; body: unknown },+ run: (core: CoreClient) => Promise<void>,+): Promise<void> {+ const server = createServer((req, res) => {+ assert.ok(req.headers["x-signature"]);+ const { status = 200, body } = response(new URL(req.url!, "http://localhost").pathname);+ res.writeHead(status, { "content-type": "application/json" });+ res.end(JSON.stringify(body));+ });+ server.listen(0, "127.0.0.1");+ await once(server, "listening");+ try {+ const address = server.address();+ assert.ok(address && typeof address !== "string");+ await run(new CoreClient(`http://127.0.0.1:${address.port}`, "synthetic-secret"));+ } finally {+ server.closeAllConnections();+ await new Promise<void>((resolve) => server.close(() => resolve()));+ }+}++for (const principal of ["[email protected]", "UQA"]) {+ test(`membership readiness waits for the exact QA identity ${principal} and persisted roster`, async () => {+ let resolutions = 0;+ let memberships = 0;+ await withMembershipServer(+ (path) => {+ if (path === "/v1/directory/resolve") {+ resolutions++;+ return { body: { matches: resolutions === 1 ? [] : [{ principalId: principal, slackId: "UQA" }] } };+ }+ assert.equal(path, `/v1/directory/channels/CNEW/members/${encodeURIComponent(principal)}`);+ return { body: { member: ++memberships === 2 } };+ },+ (core) => core.waitForChannelMembership("CNEW", "UQA", 5000),+ );+ assert.equal(resolutions, 3);+ assert.equal(memberships, 2);+ });+}++for (const body of [+ {},+ { matches: [{ principalId: "other", slackId: "UOTHER" }] },+ { matches: [{ principalId: "UQA" }, { principalId: "UQA" }] },+]) {+ test(`membership readiness rejects malformed or mismatched identity ${JSON.stringify(body)}`, async () => {+ let requests = 0;+ await withMembershipServer(+ () => {+ requests++;+ return { body };+ },+ (core) => assert.rejects(core.waitForChannelMembership("CNEW", "UQA"), /directory/),+ );+ assert.equal(requests, 1);+ });+}++for (const failure of [{ status: 403, body: {} }, { body: {} }, { body: { member: "true" } }]) {+ test(`membership readiness fails promptly on an invalid roster response ${JSON.stringify(failure)}`, async () => {+ let requests = 0;+ await withMembershipServer(+ (path) => {+ requests++;+ return path === "/v1/directory/resolve" ? { body: { matches: [{ principalId: "UQA" }] } } : failure;+ },+ (core) => assert.rejects(core.waitForChannelMembership("CNEW", "UQA")),+ );+ assert.equal(requests, 2);+ });+}++test("membership readiness has one deadline including unresolved identity and inherits cancellation", async () => {+ await withMembershipServer(+ () => ({ body: { matches: [] } }),+ async (core) => {+ await assert.rejects(core.waitForChannelMembership("CNEW", "UQA", 30), /readiness.*UQA.*CNEW/);+ await assert.rejects(+ core.withSignal(AbortSignal.timeout(30)).waitForChannelMembership("CNEW", "UQA"),+ /readiness.*UQA.*CNEW/,+ );+ },+ );+});diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.tsindex b332144d4..b0838648c 100644--- a/test/live-slack-sandbox-providers.test.ts+++ b/test/live-slack-sandbox-providers.test.ts@@ -98,6 +98,7 @@ for (const failExecution of [false, true]) {const actions: Record<string, unknown>[] = [];let expected = "";const ctx = {+ env: { qaUserId: "UQA" },marker: () => "owned-test-box",freshChannel: async () => ({id: "test",@@ -122,6 +123,11 @@ for (const failExecution of [false, true]) {? [{ id: "box", backend: "sprites", name: "owned-test-box" }]: [],}),+ waitForChannelMembership: async (channel: string, actor: string) => {+ assert.equal(channel, "test");+ assert.equal(actor, "UQA");+ assert.equal(actions.length, 0);+ },manageSandbox: async (_scope: string, body: Record<string, unknown>) => {actions.push(body);return { id: "box", backend: "sprites" };@@ -157,3 +163,26 @@ test("explicit provider selection retains strict coverage without treating deferfor (const value of ["", "sprites,", "typo", "sprites,sprites", "all,sprites", "constructor", "__proto__"])assert.throws(() => selectSandboxProviderScenarios(value));});++for (const backend of sandboxProviders) {+ test(`${backend} does not create or execute a sandbox before directory readiness`, async () => {+ const ctx = {+ env: { qaUserId: "UQA" },+ freshChannel: async () => ({ id: "test", mention: () => assert.fail("agent must not run") }),+ core: {+ withSignal() {+ return this;+ },+ listSandboxes: async () => ({ providers: [{ name: backend, actions: ["create", "retire"] }] }),+ waitForChannelMembership: async () => {+ throw new Error("directory readiness expired");+ },+ manageSandbox: () => assert.fail("sandbox must not be created"),+ },+ } as unknown as Ctx;+ await assert.rejects(+ sandboxProviderScenarios.find((s) => s.name === `sandbox-execute-${backend}`)!.run(ctx),+ /directory readiness expired/,+ );+ });+}diff --git a/test/live-slack/core.ts b/test/live-slack/core.tsindex ef5f74357..0a68cab6e 100644--- a/test/live-slack/core.ts+++ b/test/live-slack/core.ts@@ -1,3 +1,4 @@+import { setTimeout as delay } from "node:timers/promises";import { signedRequestHeaders } from "../../src/auth/source-auth-sign.ts";import { mintCapabilityToken } from "../../src/auth/capability-token.ts";import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../src/auth/portal-identity.ts";@@ -58,6 +59,40 @@ export class CoreClient {});}+ async waitForChannelMembership(channelId: string, slackId: string, timeoutMs = 60_000): Promise<void> {+ const deadline = AbortSignal.timeout(timeoutMs);+ const signal = this.requestSignal ? AbortSignal.any([deadline, this.requestSignal]) : deadline;+ const core = this.withSignal(signal);+ try {+ while (true) {+ signal.throwIfAborted();+ const { matches } = await core.request("GET", `/v1/directory/resolve?q=${encodeURIComponent(slackId)}`);+ if (!Array.isArray(matches)) throw new Error("invalid directory resolution response");+ if (matches.length) {+ if (+ matches.length !== 1 ||+ typeof matches[0]?.principalId !== "string" ||+ (matches[0].slackId !== slackId && matches[0].principalId !== slackId)+ )+ throw new Error(`directory did not resolve the exact QA Slack identity ${slackId}`);+ const { member } = await core.request(+ "GET",+ `/v1/directory/channels/${encodeURIComponent(channelId)}/members/${encodeURIComponent(matches[0].principalId)}`,+ );+ if (member === true) return;+ if (member !== false) throw new Error("invalid directory membership response");+ }+ await delay(1000, undefined, { signal });+ }+ } catch (error) {+ if (signal.aborted)+ throw new Error(`directory membership readiness timed out or was aborted for ${slackId} in ${channelId}`, {+ cause: error,+ });+ throw error;+ }+ }+listSandboxes(scopeId: string): Promise<{providers: Array<{ name: string; actions: string[] }>;sandboxes: Array<{ id: string; name: string; backend: string; state: string }>;diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.tsindex 20d3b5b08..c67e8093a 100644--- a/test/live-slack/scenarios-sandbox-providers.ts+++ b/test/live-slack/scenarios-sandbox-providers.ts@@ -55,6 +55,7 @@ export const sandboxProviderScenarios: Scenario[] = sandboxProviders.map((backenassert.ok(provider, `required sandbox provider ${backend} is unavailable`);assert.ok(provider.actions.includes("create"), `${backend} cannot create a test sandbox`);assert.ok(provider.actions.includes("retire"), `${backend} cannot clean up a test sandbox`);+ await core.waitForChannelMembership(ch.id, ctx.env.qaUserId);const name = ctx.marker();const failures: unknown[] = [];try {