Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Keep uploads usable without a default sandbox (#1670)

Joshua France183eefbf1c5d
diff --git a/src/core/attachments.ts b/src/core/attachments.ts
index 6a33e6854..92c7bcc71 100644
--- a/src/core/attachments.ts
+++ b/src/core/attachments.ts
@@ -16,6 +16,7 @@ import { swallowAs } from "../util/errors.ts";
import { hashId } from "../util/crypto.ts";
import type { SecurityScreenVerdict } from "../security/security-posture.ts";
import { downscaleVisionImage } from "./image-downscale.ts";
+import { NoDefaultSandboxError } from "../sandbox/sandbox-routing.ts";
export const INBOX_DIR = "inbox";
export const SHARED_DIR = "shared";
@@ -202,18 +203,23 @@ function shownToModel(meta: AttachmentMeta): boolean {
return isVisionAttachment(meta) && meta.sizeBytes > 0 && meta.sizeBytes <= MAX_VISION_IMAGE_BYTES;
}
-export function inboundManifest(metas: AttachmentMeta[], inboxDir = INBOX_DIR): string {
+export function inboundManifest(metas: AttachmentMeta[], inboxDir = INBOX_DIR, unstaged?: Set<string>): string {
if (!metas.length) return "";
const list = metas
.map(
(m) =>
- `- ${inboxDir}/${m.name} (${m.mimetype}, ${m.sizeBytes} bytes)${m.author ? ` — shared by ${m.author}` : ""}`,
+ `- ${inboxDir}/${m.name} (${m.mimetype}, ${m.sizeBytes} bytes)${m.author ? ` — shared by ${m.author}` : ""}${
+ unstaged?.has(m.name)
+ ? ` — saved as file ${m.artifactId}, not yet on a computer (no default sandbox); if you need it on disk, select or create a sandbox with set_default, then download GET $AGENT_API_URL/v1/files/${m.artifactId}/content (header x-agent-capability: $AGENT_API_TOKEN) to a path you choose`
+ : ""
+ }`,
)
.join("\n");
const noun = metas.length === 1 ? "file" : "files";
+ const where = unstaged?.size ? `for ./${inboxDir}/` : `available in ./${inboxDir}/`;
const lead = metas.some((m) => m.author)
- ? `${metas.length} ${noun} shared in this conversation, available in ./${inboxDir}/:`
- : `The user shared ${metas.length} ${noun}, available in ./${inboxDir}/:`;
+ ? `${metas.length} ${noun} shared in this conversation, ${where}:`
+ : `The user shared ${metas.length} ${noun}, ${where}:`;
return `${lead}\n${list}`;
}
@@ -301,7 +307,7 @@ export function senderNote(name: string | undefined): string {
export async function materializeInbound(
sandbox: Sandbox,
- handle: SandboxHandle,
+ handle: SandboxHandle | (() => Promise<SandboxHandle>),
attachments: IncomingAttachment[],
transfer: BlobTransferStore,
register?: ArtifactRegistration,
@@ -318,7 +324,15 @@ export async function materializeInbound(
unavailable: string[];
blocked: string[];
unscreened: string[];
+ unstaged?: Set<string>;
}> {
+ let target: Promise<SandboxHandle | null> | undefined;
+ const computer = () =>
+ (target ??= (typeof handle === "function" ? handle() : Promise.resolve(handle)).catch((e: unknown) => {
+ if (e instanceof NoDefaultSandboxError) return null;
+ throw e;
+ }));
+ const unstaged = new Set<string>();
const metas: AttachmentMeta[] = [];
const images: InboundImage[] = [];
const tooMany: string[] = [];
@@ -349,10 +363,18 @@ export async function materializeInbound(
}
if (!verdict || verdict.unscreened) unscreened.push(name);
}
- await sandbox.writeFileBytes(handle, `${inboxDir}/${name}`, bytes);
+ const box = await computer();
+ if (box) await sandbox.writeFileBytes(box, `${inboxDir}/${name}`, bytes);
const registered = register
? await registerArtifact(register, "in", metas.length, name, mimetype, bytes)
: undefined;
+ if (!box) {
+ if (!registered) {
+ unavailable.push(name);
+ continue;
+ }
+ unstaged.add(name);
+ }
metas.push({
name,
mimetype,
@@ -372,7 +394,7 @@ export async function materializeInbound(
});
}
}
- return { metas, images, tooMany, unavailable, blocked, unscreened };
+ return { metas, images, tooMany, unavailable, blocked, unscreened, ...(unstaged.size ? { unstaged } : {}) };
}
const DELIVERY_NOTE_PREFIX = "[files delivered to the conversation: ";
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts
index 1e8b0a4d7..12cfc2381 100644
--- a/src/core/orchestrator.ts
+++ b/src/core/orchestrator.ts
@@ -2782,7 +2782,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
input.attachments?.length && !strictReadOnly
? await materializeInbound(
deps.sandbox,
- await provision(),
+ provision,
input.attachments,
blobTransfer,
fileRegistration,
@@ -2799,7 +2799,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
: undefined,
)
: { metas: [], images: [], tooMany: [], unavailable: [], blocked: [], unscreened: [] };
- const manifest = inboundManifest(inbound.metas, turnInboxDir);
+ const manifest = inboundManifest(inbound.metas, turnInboxDir, inbound.unstaged);
const inboundIssues = inboundIssueList({
tooMany: inbound.tooMany,
unavailable: inbound.unavailable,
@@ -3445,7 +3445,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
? { metas: [], images: [], tooMany: [], unavailable: [], blocked: [], unscreened: [] }
: await materializeInbound(
deps.sandbox,
- await provision(),
+ provision,
request.attachments,
blobTransfer,
{ ...fileRegistration, seed },
@@ -3489,7 +3489,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
return {
text: [
text,
- inboundManifest(received.metas, inboxDir),
+ inboundManifest(received.metas, inboxDir, received.unstaged),
...steeredDocuments.notices,
issues.length ? fileEventPayload("in", issues).text : "",
securityPolicy.inboundScreening === "external" &&
diff --git a/src/sandbox/sandbox-routing.ts b/src/sandbox/sandbox-routing.ts
index 0eff8d173..e1580d3f4 100644
--- a/src/sandbox/sandbox-routing.ts
+++ b/src/sandbox/sandbox-routing.ts
@@ -21,8 +21,13 @@ export type SandboxBackendName =
export type SandboxScopeDefaults = Partial<Record<ScopeKind, SandboxBackendName>>;
-const NO_DEFAULT_SANDBOX =
- "this scope has no default sandbox; use sandbox list, create, set_default, then retry before reporting blocked";
+export class NoDefaultSandboxError extends Error {
+ constructor() {
+ super(
+ "this scope has no default sandbox; use sandbox list, create, set_default, then retry before reporting blocked",
+ );
+ }
+}
export function sandboxDefaultForScope(
scope: string | undefined,
@@ -96,7 +101,7 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {
async function computerTarget(scopeId: string): Promise<{ sandbox: Sandbox; scopeId: string; resourceId?: string }> {
const resource = await opts.resources?.resolve(scopeId);
- if (resource === null) throw new Error(NO_DEFAULT_SANDBOX);
+ if (resource === null) throw new NoDefaultSandboxError();
if (resource) {
const sandbox = backends[resource.backend];
if (!sandbox) throw new Error(`sandbox backend unavailable: ${resource.backend}`);
@@ -166,7 +171,7 @@ export function createSandboxRouter(opts: RoutingSandboxOptions): Sandbox {
if (provOpts?.sandboxId) resource = await opts.resources?.get(provOpts.sandboxId);
else if (!provOpts?.scratch) resource = await opts.resources?.resolve(scope);
if (provOpts?.sandboxId && !resource) throw new Error("sandbox inventory unavailable");
- if (resource === null) throw new Error(NO_DEFAULT_SANDBOX);
+ if (resource === null) throw new NoDefaultSandboxError();
if (resource) {
const sandbox = backends[resource.backend];
if (!sandbox) throw new Error(`sandbox backend unavailable: ${resource.backend}`);
diff --git a/test/file-sharing.test.ts b/test/file-sharing.test.ts
index 4e14df1f3..823dd2957 100644
--- a/test/file-sharing.test.ts
+++ b/test/file-sharing.test.ts
@@ -4,6 +4,7 @@ import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Sandbox, SandboxHandle } from "../src/sandbox/sandbox.ts";
+import { NoDefaultSandboxError } from "../src/sandbox/sandbox-routing.ts";
import type { GrantedHandle, IncomingAttachment } from "../src/types.ts";
import { createSpritesSandbox } from "../src/sandbox/sprites-sandbox.ts";
import { createLocalWorkspaceStore } from "../src/workspace/workspace-store.ts";
@@ -627,3 +628,50 @@ test("a binary file round-trips through the sandbox (base64-over-exec) without u
const read = await sandbox.readFileBytes(handle, "keep.bin");
assert.deepEqual(new Uint8Array(read!), raw);
});
+
+test("materializeInbound keeps uploads registered when the scope has no default sandbox", async () => {
+ const { sandbox, files } = fakeSandbox();
+ const transfer = createMemoryBlobTransferStore();
+ const put = async () => ({ created: true });
+ const register = { seed: "s", ownerScopeId: "channel:C1", createdBy: "U1", store: { put } } as never;
+ const noComputer = () => Promise.reject(new NoDefaultSandboxError());
+ const got = await materializeInbound(
+ sandbox,
+ noComputer,
+ [await inFile(transfer, "a b.txt", "hi")],
+ transfer,
+ register,
+ );
+ assert.equal(files.size, 0);
+ assert.equal(got.metas.length, 1);
+ assert.ok(got.unstaged?.has("a b.txt"));
+ const manifest = inboundManifest(got.metas, "inbox", got.unstaged);
+ assert.match(manifest, new RegExp(`/v1/files/${got.metas[0]!.artifactId}/content`));
+ assert.doesNotMatch(manifest, /available in/);
+});
+
+test("materializeInbound drops an unstaged upload it could not register, and rethrows other provision failures", async () => {
+ const { sandbox } = fakeSandbox();
+ const transfer = createMemoryBlobTransferStore();
+ const put = async () => {
+ throw new Error("store down");
+ };
+ const register = { seed: "s", ownerScopeId: "channel:C1", createdBy: "U1", store: { put } } as never;
+ const got = await materializeInbound(
+ sandbox,
+ () => Promise.reject(new NoDefaultSandboxError()),
+ [await inFile(transfer, "a.txt", "hi")],
+ transfer,
+ register,
+ );
+ assert.deepEqual([got.metas.length, got.unavailable], [0, ["a.txt"]]);
+ await assert.rejects(
+ materializeInbound(
+ sandbox,
+ () => Promise.reject(new Error("provider down")),
+ [await inFile(transfer, "b.txt", "x")],
+ transfer,
+ ),
+ /provider down/,
+ );
+});
diff --git a/test/upload-no-default-native.test.ts b/test/upload-no-default-native.test.ts
new file mode 100644
index 000000000..fe2b6a125
--- /dev/null
+++ b/test/upload-no-default-native.test.ts
@@ -0,0 +1,93 @@
+import "./support/auto-fake-sprites.ts";
+import { mock, test } from "node:test";
+import assert from "node:assert/strict";
+import * as mockHarness from "../src/harness/mock-harness.ts";
+import type { HarnessTurnInput, HarnessTurnResult } from "../src/harness/harness.ts";
+import type { TurnRequest } from "../src/types.ts";
+import { testConfig } from "./support/test-config.ts";
+
+let exercise: ((turn: HarnessTurnInput) => Promise<HarnessTurnResult | void>) | undefined;
+mock.module("../src/harness/mock-harness.ts", {
+ namedExports: {
+ ...mockHarness,
+ createMockHarness: () => {
+ const harness = mockHarness.createMockHarness();
+ const run = harness.turns.runTurn;
+ harness.turns.runTurn = async (turn) => (exercise ? ((await exercise(turn)) ?? { reply: "Done" }) : run(turn));
+ return harness;
+ },
+ },
+});
+const { buildApp } = await import("../src/wiring.ts");
+
+const PNG = Buffer.from(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
+ "base64",
+);
+
+async function scopeWithoutDefault() {
+ const built = buildApp(testConfig({ sandboxResourcesEnabled: true, eagerProvisionEnabled: false }));
+ await built.directory.replace([{ principalId: "U1", displayName: "Synthetic User", type: "internal" }]);
+ await built.directory.replaceChannels(
+ [{ channelId: "C1", name: "synthetic-upload", isPrivate: true }],
+ [{ channelId: "C1", principalId: "U1" }],
+ );
+ await built.sandboxResources.initialize();
+ const base: Omit<TurnRequest, "text"> = {
+ surface: "slack",
+ actor: { externalId: "U1" },
+ conversation: { kind: "channel", threadRef: "ch:C1:native", channelRef: "C1", audience: [{ externalId: "U1" }] },
+ };
+ const files = async () => [
+ { name: "shot.png", mimetype: "image/png", ...(await built.blobTransfer.put(PNG)) },
+ { name: "notes.txt", mimetype: "text/plain", ...(await built.blobTransfer.put(Buffer.from("SYNTHETIC-42"))) },
+ ];
+ return { built, base, files };
+}
+
+async function artifactText(built: Awaited<ReturnType<typeof scopeWithoutDefault>>["built"], id: string) {
+ const opened = await built.files.open(id);
+ assert.ok(opened);
+ const chunks: Buffer[] = [];
+ for await (const chunk of opened.stream) chunks.push(Buffer.from(chunk));
+ return Buffer.concat(chunks);
+}
+
+test("without a default sandbox, native images and documents still reach the harness", async () => {
+ const { built, base, files } = await scopeWithoutDefault();
+ let seen: HarnessTurnInput | undefined;
+ exercise = async (turn) => {
+ seen = turn;
+ };
+ try {
+ const result = await built.app.turn({ ...base, text: "look", attachments: await files() });
+ assert.equal(result.status, "ok");
+ assert.equal(await built.sandboxResources.resolve("channel:C1"), null);
+ assert.equal(Buffer.from(seen!.images![0]!.dataBase64, "base64").length > 0, true);
+ assert.equal(Buffer.from(seen!.documents![0]!.dataBase64, "base64").toString(), "SYNTHETIC-42");
+ assert.deepEqual(await artifactText(built, seen!.images![0]!.artifactId!), PNG);
+ assert.equal((await artifactText(built, seen!.documents![0]!.artifactId!)).toString(), "SYNTHETIC-42");
+ } finally {
+ exercise = undefined;
+ }
+});
+
+test("without a default sandbox, a steered upload is registered and surfaced without a computer", async () => {
+ const { built, base, files } = await scopeWithoutDefault();
+ let steered: Awaited<ReturnType<NonNullable<HarnessTurnInput["prepareSteer"]>>> | undefined;
+ exercise = async (turn) => {
+ steered = await turn.prepareSteer!("also this", { ...base, text: "also this", attachments: await files() });
+ };
+ try {
+ const result = await built.app.turn({ ...base, text: "start" });
+ assert.equal(result.status, "ok");
+ assert.equal(await built.sandboxResources.resolve("channel:C1"), null);
+ assert.match(steered!.text, /not yet on a computer/);
+ assert.equal(steered!.images?.length, 1);
+ assert.equal(Buffer.from(steered!.documents![0]!.dataBase64, "base64").toString(), "SYNTHETIC-42");
+ const doc = steered!.attachments!.find((a) => a.name === "notes.txt")!;
+ assert.equal((await artifactText(built, doc.artifactId!)).toString(), "SYNTHETIC-42");
+ } finally {
+ exercise = undefined;
+ }
+});
diff --git a/test/upload-without-default-sandbox.test.ts b/test/upload-without-default-sandbox.test.ts
new file mode 100644
index 000000000..be7efe975
--- /dev/null
+++ b/test/upload-without-default-sandbox.test.ts
@@ -0,0 +1,39 @@
+import "./support/auto-fake-sprites.ts";
+import test from "node:test";
+import assert from "node:assert/strict";
+import { buildApp } from "../src/wiring.ts";
+import { testConfig } from "./support/test-config.ts";
+import type { TurnRequest } from "../src/types.ts";
+
+test("an upload in a scope without a default sandbox still reaches the agent", async () => {
+ const built = buildApp(testConfig({ sandboxResourcesEnabled: true, eagerProvisionEnabled: false }));
+ await built.directory.replace([{ principalId: "U1", displayName: "Synthetic User", type: "internal" }]);
+ await built.directory.replaceChannels(
+ [{ channelId: "C1", name: "synthetic-upload", isPrivate: true }],
+ [{ channelId: "C1", principalId: "U1" }],
+ );
+ await built.sandboxResources.initialize();
+ const request: Omit<TurnRequest, "text"> = {
+ surface: "slack",
+ actor: { externalId: "U1" },
+ conversation: { kind: "channel", threadRef: "ch:C1:upload", channelRef: "C1", audience: [{ externalId: "U1" }] },
+ };
+ const first = await built.app.turn({ ...request, text: "I will share a document" });
+ assert.equal(first.status, "ok");
+ const sessionId = first.sessionId!;
+ const blob = await built.blobTransfer.put(Buffer.from("Synthetic upload sample"));
+ const before = (await built.sessions.listLlmRequests(sessionId)).length;
+ const upload = await built.app.turn({
+ ...request,
+ text: "Please read this document",
+ attachments: [{ name: "sample.txt", mimetype: "text/plain", sizeBytes: blob.sizeBytes, blobId: blob.blobId }],
+ });
+ assert.equal(upload.status, "ok", upload.status === "ok" ? undefined : upload.reason);
+ assert.ok((await built.sessions.listLlmRequests(sessionId)).length > before);
+ assert.equal(await built.sandboxResources.resolve("channel:C1"), null);
+ const entries = await built.sessions.getEntries(sessionId);
+ const text = JSON.stringify(entries.filter((e) => e.type === "user"));
+ assert.match(text, /sample\.txt/);
+ assert.match(text, /not yet on a computer/);
+ assert.match(text, /\/v1\/files\/[^/]+\/content/);
+});