Multiplayer agent harness for work.
Keep uploads usable without a default sandbox (#1670)
Joshua France183eefbf1c5d
diff --git a/src/core/attachments.ts b/src/core/attachments.tsindex 6a33e6854..92c7bcc71 100644--- a/src/core/attachments.ts+++ b/src/core/attachments.ts@@ -16,6 +16,7 @@ import { swallowAs } from "../util/errors.ts";import { hashId } from "../util/crypto.ts";import type { SecurityScreenVerdict } from "../security/security-posture.ts";import { downscaleVisionImage } from "./image-downscale.ts";+import { NoDefaultSandboxError } from "../sandbox/sandbox-routing.ts";export const INBOX_DIR = "inbox";export const SHARED_DIR = "shared";@@ -202,18 +203,23 @@ function shownToModel(meta: AttachmentMeta): boolean {return isVisionAttachment(meta) && meta.sizeBytes > 0 && meta.sizeBytes <= MAX_VISION_IMAGE_BYTES;}-export function inboundManifest(metas: AttachmentMeta[], inboxDir = INBOX_DIR): string {+export function inboundManifest(metas: AttachmentMeta[], inboxDir = INBOX_DIR, unstaged?: Set<string>): string {if (!metas.length) return "";const list = metas.map((m) =>- `- ${inboxDir}/${m.name} (${m.mimetype}, ${m.sizeBytes} bytes)${m.author ? ` — shared by ${m.author}` : ""}`,+ `- ${inboxDir}/${m.name} (${m.mimetype}, ${m.sizeBytes} bytes)${m.author ? ` — shared by ${m.author}` : ""}${+ unstaged?.has(m.name)+ ? ` — saved as file ${m.artifactId}, not yet on a computer (no default sandbox); if you need it on disk, select or create a sandbox with set_default, then download GET $AGENT_API_URL/v1/files/${m.artifactId}/content (header x-agent-capability: $AGENT_API_TOKEN) to a path you choose`+ : ""+ }`,).join("\n");const noun = metas.length === 1 ? "file" : "files";+ const where = unstaged?.size ? `for ./${inboxDir}/` : `available in ./${inboxDir}/`;const lead = metas.some((m) => m.author)- ? `${metas.length} ${noun} shared in this conversation, available in ./${inboxDir}/:`- : `The user shared ${metas.length} ${noun}, available in ./${inboxDir}/:`;+ ? `${metas.length} ${noun} shared in this conversation, ${where}:`+ : `The user shared ${metas.length} ${noun}, ${where}:`;return `${lead}\n${list}`;}@@ -301,7 +307,7 @@ export function senderNote(name: string | undefined): string {export async function materializeInbound(sandbox: Sandbox,- handle: SandboxHandle,+ handle: SandboxHandle | (() => Promise<SandboxHandle>),attachments: IncomingAttachment[],transfer: BlobTransferStore,register?: ArtifactRegistration,@@ -318,7 +324,15 @@ export async function materializeInbound(unavailable: string[];blocked: string[];unscreened: string[];+ unstaged?: Set<string>;}> {+ let target: Promise<SandboxHandle | null> | undefined;+ const computer = () =>+ (target ??= (typeof handle === "function" ? handle() : Promise.resolve(handle)).catch((e: unknown) => {+ if (e instanceof NoDefaultSandboxError) return null;+ throw e;+ }));+ const unstaged = new Set<string>();const metas: AttachmentMeta[] = [];const images: InboundImage[] = [];const tooMany: string[] = [];@@ -349,10 +363,18 @@ export async function materializeInbound(}if (!verdict || verdict.unscreened) unscreened.push(name);}- await sandbox.writeFileBytes(handle, `${inboxDir}/${name}`, bytes);+ const box = await computer();+ if (box) await sandbox.writeFileBytes(box, `${inboxDir}/${name}`, bytes);const registered = register? await registerArtifact(register, "in", metas.length, name, mimetype, bytes): undefined;+ if (!box) {+ if (!registered) {+ unavailable.push(name);+ continue;+ }+ unstaged.add(name);+ }metas.push({name,mimetype,@@ -372,7 +394,7 @@ export async function materializeInbound(});}}- return { metas, images, tooMany, unavailable, blocked, unscreened };+ return { metas, images, tooMany, unavailable, blocked, unscreened, ...(unstaged.size ? { unstaged } : {}) };}const DELIVERY_NOTE_PREFIX = "[files delivered to the conversation: ";diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.tsindex 1e8b0a4d7..12cfc2381 100644--- a/src/core/orchestrator.ts+++ b/src/core/orchestrator.ts@@ -2782,7 +2782,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {input.attachments?.length && !strictReadOnly? await materializeInbound(deps.sandbox,- await provision(),+ provision,input.attachments,blobTransfer,fileRegistration,@@ -2799,7 +2799,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {: undefined,): { metas: [], images: [], tooMany: [], unavailable: [], blocked: [], unscreened: [] };- const manifest = inboundManifest(inbound.metas, turnInboxDir);+ const manifest = inboundManifest(inbound.metas, turnInboxDir, inbound.unstaged);const inboundIssues = inboundIssueList({tooMany: inbound.tooMany,unavailable: inbound.unavailable,@@ -3445,7 +3445,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {? { metas: [], images: [], tooMany: [], unavailable: [], blocked: [], unscreened: [] }: await materializeInbound(deps.sandbox,- await provision(),+ provision,request.attachments,blobTransfer,{ ...fileRegistration, seed },@@ -3489,7 +3489,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {return {text: [text,- inboundManifest(received.metas, inboxDir),+ inboundManifest(received.metas, inboxDir, received.unstaged),...steeredDocuments.notices,issues.length ? fileEventPayload("in", issues).text : "",securityPolicy.inboundScreening === "external" &&diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.tsindex 0eff8d173..e1580d3f4 100644--- a/src/sandbox/sandbox-routing.ts+++ b/src/sandbox/sandbox-routing.ts@@ -21,8 +21,13 @@ export type SandboxBackendName =export type SandboxScopeDefaults = Partial<Record<ScopeKind, SandboxBackendName>>;-const NO_DEFAULT_SANDBOX =- "this scope has no default sandbox; use sandbox list, create, set_default, then retry before reporting blocked";+export class NoDefaultSandboxError extends Error {+ constructor() {+ super(+ "this scope has no default sandbox; use sandbox list, create, set_default, then retry before reporting blocked",+ );+ }+}export function sandboxDefaultForScope(scope: string | undefined,@@ -96,7 +101,7 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {async function computerTarget(scopeId: string): Promise<{ sandbox: Sandbox; scopeId: string; resourceId?: string }> {const resource = await opts.resources?.resolve(scopeId);- if (resource === null) throw new Error(NO_DEFAULT_SANDBOX);+ if (resource === null) throw new NoDefaultSandboxError();if (resource) {const sandbox = backends[resource.backend];if (!sandbox) throw new Error(`sandbox backend unavailable: ${resource.backend}`);@@ -166,7 +171,7 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {if (provOpts?.sandboxId) resource = await opts.resources?.get(provOpts.sandboxId);else if (!provOpts?.scratch) resource = await opts.resources?.resolve(scope);if (provOpts?.sandboxId && !resource) throw new Error("sandbox inventory unavailable");- if (resource === null) throw new Error(NO_DEFAULT_SANDBOX);+ if (resource === null) throw new NoDefaultSandboxError();if (resource) {const sandbox = backends[resource.backend];if (!sandbox) throw new Error(`sandbox backend unavailable: ${resource.backend}`);diff --git a/test/file-sharing.test.ts b/test/file-sharing.test.tsindex 4e14df1f3..823dd2957 100644--- a/test/file-sharing.test.ts+++ b/test/file-sharing.test.ts@@ -4,6 +4,7 @@ import { mkdtempSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import type { Sandbox, SandboxHandle } from "../src/sandbox/sandbox.ts";+import { NoDefaultSandboxError } from "../src/sandbox/sandbox-routing.ts";import type { GrantedHandle, IncomingAttachment } from "../src/types.ts";import { createSpritesSandbox } from "../src/sandbox/sprites-sandbox.ts";import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";@@ -627,3 +628,50 @@ test("a binary file round-trips through the sandbox (base64-over-exec) without uconst read = await sandbox.readFileBytes(handle, "keep.bin");assert.deepEqual(new Uint8Array(read!), raw);});++test("materializeInbound keeps uploads registered when the scope has no default sandbox", async () => {+ const { sandbox, files } = fakeSandbox();+ const transfer = createMemoryBlobTransferStore();+ const put = async () => ({ created: true });+ const register = { seed: "s", ownerScopeId: "channel:C1", createdBy: "U1", store: { put } } as never;+ const noComputer = () => Promise.reject(new NoDefaultSandboxError());+ const got = await materializeInbound(+ sandbox,+ noComputer,+ [await inFile(transfer, "a b.txt", "hi")],+ transfer,+ register,+ );+ assert.equal(files.size, 0);+ assert.equal(got.metas.length, 1);+ assert.ok(got.unstaged?.has("a b.txt"));+ const manifest = inboundManifest(got.metas, "inbox", got.unstaged);+ assert.match(manifest, new RegExp(`/v1/files/${got.metas[0]!.artifactId}/content`));+ assert.doesNotMatch(manifest, /available in/);+});++test("materializeInbound drops an unstaged upload it could not register, and rethrows other provision failures", async () => {+ const { sandbox } = fakeSandbox();+ const transfer = createMemoryBlobTransferStore();+ const put = async () => {+ throw new Error("store down");+ };+ const register = { seed: "s", ownerScopeId: "channel:C1", createdBy: "U1", store: { put } } as never;+ const got = await materializeInbound(+ sandbox,+ () => Promise.reject(new NoDefaultSandboxError()),+ [await inFile(transfer, "a.txt", "hi")],+ transfer,+ register,+ );+ assert.deepEqual([got.metas.length, got.unavailable], [0, ["a.txt"]]);+ await assert.rejects(+ materializeInbound(+ sandbox,+ () => Promise.reject(new Error("provider down")),+ [await inFile(transfer, "b.txt", "x")],+ transfer,+ ),+ /provider down/,+ );+});diff --git a/test/upload-no-default-native.test.ts b/test/upload-no-default-native.test.tsnew file mode 100644index 000000000..fe2b6a125--- /dev/null+++ b/test/upload-no-default-native.test.ts@@ -0,0 +1,93 @@+import "./support/auto-fake-sprites.ts";+import { mock, test } from "node:test";+import assert from "node:assert/strict";+import * as mockHarness from "../src/harness/mock-harness.ts";+import type { HarnessTurnInput, HarnessTurnResult } from "../src/harness/harness.ts";+import type { TurnRequest } from "../src/types.ts";+import { testConfig } from "./support/test-config.ts";++let exercise: ((turn: HarnessTurnInput) => Promise<HarnessTurnResult | void>) | undefined;+mock.module("../src/harness/mock-harness.ts", {+ namedExports: {+ ...mockHarness,+ createMockHarness: () => {+ const harness = mockHarness.createMockHarness();+ const run = harness.turns.runTurn;+ harness.turns.runTurn = async (turn) => (exercise ? ((await exercise(turn)) ?? { reply: "Done" }) : run(turn));+ return harness;+ },+ },+});+const { buildApp } = await import("../src/wiring.ts");++const PNG = Buffer.from(+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",+ "base64",+);++async function scopeWithoutDefault() {+ const built = buildApp(testConfig({ sandboxResourcesEnabled: true, eagerProvisionEnabled: false }));+ await built.directory.replace([{ principalId: "U1", displayName: "Synthetic User", type: "internal" }]);+ await built.directory.replaceChannels(+ [{ channelId: "C1", name: "synthetic-upload", isPrivate: true }],+ [{ channelId: "C1", principalId: "U1" }],+ );+ await built.sandboxResources.initialize();+ const base: Omit<TurnRequest, "text"> = {+ surface: "slack",+ actor: { externalId: "U1" },+ conversation: { kind: "channel", threadRef: "ch:C1:native", channelRef: "C1", audience: [{ externalId: "U1" }] },+ };+ const files = async () => [+ { name: "shot.png", mimetype: "image/png", ...(await built.blobTransfer.put(PNG)) },+ { name: "notes.txt", mimetype: "text/plain", ...(await built.blobTransfer.put(Buffer.from("SYNTHETIC-42"))) },+ ];+ return { built, base, files };+}++async function artifactText(built: Awaited<ReturnType<typeof scopeWithoutDefault>>["built"], id: string) {+ const opened = await built.files.open(id);+ assert.ok(opened);+ const chunks: Buffer[] = [];+ for await (const chunk of opened.stream) chunks.push(Buffer.from(chunk));+ return Buffer.concat(chunks);+}++test("without a default sandbox, native images and documents still reach the harness", async () => {+ const { built, base, files } = await scopeWithoutDefault();+ let seen: HarnessTurnInput | undefined;+ exercise = async (turn) => {+ seen = turn;+ };+ try {+ const result = await built.app.turn({ ...base, text: "look", attachments: await files() });+ assert.equal(result.status, "ok");+ assert.equal(await built.sandboxResources.resolve("channel:C1"), null);+ assert.equal(Buffer.from(seen!.images![0]!.dataBase64, "base64").length > 0, true);+ assert.equal(Buffer.from(seen!.documents![0]!.dataBase64, "base64").toString(), "SYNTHETIC-42");+ assert.deepEqual(await artifactText(built, seen!.images![0]!.artifactId!), PNG);+ assert.equal((await artifactText(built, seen!.documents![0]!.artifactId!)).toString(), "SYNTHETIC-42");+ } finally {+ exercise = undefined;+ }+});++test("without a default sandbox, a steered upload is registered and surfaced without a computer", async () => {+ const { built, base, files } = await scopeWithoutDefault();+ let steered: Awaited<ReturnType<NonNullable<HarnessTurnInput["prepareSteer"]>>> | undefined;+ exercise = async (turn) => {+ steered = await turn.prepareSteer!("also this", { ...base, text: "also this", attachments: await files() });+ };+ try {+ const result = await built.app.turn({ ...base, text: "start" });+ assert.equal(result.status, "ok");+ assert.equal(await built.sandboxResources.resolve("channel:C1"), null);+ assert.match(steered!.text, /not yet on a computer/);+ assert.equal(steered!.images?.length, 1);+ assert.equal(Buffer.from(steered!.documents![0]!.dataBase64, "base64").toString(), "SYNTHETIC-42");+ const doc = steered!.attachments!.find((a) => a.name === "notes.txt")!;+ assert.equal((await artifactText(built, doc.artifactId!)).toString(), "SYNTHETIC-42");+ } finally {+ exercise = undefined;+ }+});diff --git a/test/upload-without-default-sandbox.test.ts b/test/upload-without-default-sandbox.test.tsnew file mode 100644index 000000000..be7efe975--- /dev/null+++ b/test/upload-without-default-sandbox.test.ts@@ -0,0 +1,39 @@+import "./support/auto-fake-sprites.ts";+import test from "node:test";+import assert from "node:assert/strict";+import { buildApp } from "../src/wiring.ts";+import { testConfig } from "./support/test-config.ts";+import type { TurnRequest } from "../src/types.ts";++test("an upload in a scope without a default sandbox still reaches the agent", async () => {+ const built = buildApp(testConfig({ sandboxResourcesEnabled: true, eagerProvisionEnabled: false }));+ await built.directory.replace([{ principalId: "U1", displayName: "Synthetic User", type: "internal" }]);+ await built.directory.replaceChannels(+ [{ channelId: "C1", name: "synthetic-upload", isPrivate: true }],+ [{ channelId: "C1", principalId: "U1" }],+ );+ await built.sandboxResources.initialize();+ const request: Omit<TurnRequest, "text"> = {+ surface: "slack",+ actor: { externalId: "U1" },+ conversation: { kind: "channel", threadRef: "ch:C1:upload", channelRef: "C1", audience: [{ externalId: "U1" }] },+ };+ const first = await built.app.turn({ ...request, text: "I will share a document" });+ assert.equal(first.status, "ok");+ const sessionId = first.sessionId!;+ const blob = await built.blobTransfer.put(Buffer.from("Synthetic upload sample"));+ const before = (await built.sessions.listLlmRequests(sessionId)).length;+ const upload = await built.app.turn({+ ...request,+ text: "Please read this document",+ attachments: [{ name: "sample.txt", mimetype: "text/plain", sizeBytes: blob.sizeBytes, blobId: blob.blobId }],+ });+ assert.equal(upload.status, "ok", upload.status === "ok" ? undefined : upload.reason);+ assert.ok((await built.sessions.listLlmRequests(sessionId)).length > before);+ assert.equal(await built.sandboxResources.resolve("channel:C1"), null);+ const entries = await built.sessions.getEntries(sessionId);+ const text = JSON.stringify(entries.filter((e) => e.type === "user"));+ assert.match(text, /sample\.txt/);+ assert.match(text, /not yet on a computer/);+ assert.match(text, /\/v1\/files\/[^/]+\/content/);+});