Multiplayer agent harness for work.
Gate swarms per person and add a shared board and worker controls (#1770)
Joshua France7df971444d68
diff --git a/docs/swarms.md b/docs/swarms.mdindex 003ff5059..cc6a8e497 100644--- a/docs/swarms.md+++ b/docs/swarms.md@@ -16,6 +16,12 @@ credentials and deployment-specific settings. Workers use the provider of the scretirement. That choice is stored once; later default changes never move workersto another provider, and the selected computer's files are never copied.+Swarms are also gated per person by the `swarms` feature flag, off by default.+Enable it for a person's personal scope (for example `personal:[email protected]`)+or the org scope in the admin feature-flag table. Without it the swarm routes refuse+the caller, `/v1/apis` omits them, and existing swarms owned by that person stop+reconciling and delivering. `SWARMS_ENABLED=false` remains the deployment kill switch.+The `swarms` durable-map table has its own registered migration,`durable-map/swarms/0001`. Apply registered migrations through the normal QM deploypath before starting workers. Memory stores implement the same contract for tests;@@ -77,8 +83,14 @@ grants belong only to the session that received them, not its workers. A computeis not an authorization principal. Existing provider persistence behavior isunchanged; swarms add no filesystem snapshots, immutable copies, or restores.-To add a shared forum, supply `forumSandboxId` naming an existing authorized sandbox-in the same scope. Every worker still gets its own blank private computer.+Every swarm gets one shared board computer in the swarm scope, created with the+first worker. `inspect` returns it as `board.sandboxId`, and every worker carries+it as `forumSandboxId`; the root session and all workers reach it with+`execute`'s `sandbox_id`. Board creation is part of each worker's provisioning and+fails that worker like any other provisioning failure. The board outlives worker+cleanup so its files remain available to the root. To use an existing computer instead,+supply `forumSandboxId` naming an authorized sandbox in the same scope; no board is+then created. Every worker still gets its own blank private computer.The forum ID appears in peer metadata and the worker prompt; select it explicitlywith `execute`'s `sandbox_id` for commands that should use the shared computer.This is not a new filesystem synchronization feature. Workers using a forum share@@ -191,11 +203,27 @@ resource operations exhaust only this instance's resource capacity until a slotreleased. Pending selection remains single-flight until its underlying databasequery settles, even when a sweep reports a selection timeout.+### Worker controls++`POST` with `{action:"control",memberId,state}` sets a worker to `paused`,+`active`, or `stopped`. Humans in the swarm may control any worker; an agent may+control only itself or its descendants. The root session cannot be controlled.+Controls apply to descendants too. Pausing holds new notifications as pending+until the worker is resumed; runs already queued continue. Stopping is terminal:+pending notifications fail, unstarted queued runs are withdrawn, running turns are+aborted, and later turns in that worker session are refused. `inspect` reports each+peer's `control`. The web UI shows a Swarm strip on swarm sessions: a collapsed summary of+active, starting, paused, stopped, and failed workers; when expanded, workers+grouped under their parent (20 per group, with "show more"); per-worker Pause,+Resume, and Stop; and Pause all, Resume all, and Stop all (with a confirm), which+apply this action to the root's direct children so it cascades. The strip+refreshes every few seconds only while workers are active and the tab is visible.+## Defaults and safety bounds| Setting | Default | Maximum || ------------------------------------------------ | -------------------- | ----------- |-| `agents` (root and failed reservations included) | 32 | 64 |+| `agents` (root and failed reservations included) | 32 | 256 || `depth` below root | 4 | 8 || `spawnRequests` | 32 | 64 || `messages` including initial work | 128 | 256 |diff --git a/plugins/web-ui/server/index.ts b/plugins/web-ui/server/index.tsindex 86c208825..c0e2f53d1 100644--- a/plugins/web-ui/server/index.ts+++ b/plugins/web-ui/server/index.ts@@ -1942,6 +1942,21 @@ const apiRoutes: readonly WebRoute[] = [);},},+ {+ method: "GET",+ path: "/api/sessions/:id/swarm",+ handle: async (c) => relayCore(c.res, "GET", `/v1/sessions/${encodeURIComponent(c.params.id!)}/swarm`),+ },+ {+ method: "POST",+ path: "/api/sessions/:id/swarm",+ handle: async (c) => {+ const body = await readJson<{ action?: unknown }>(c.req, c.res, false);+ if (!body) return;+ if (body.action !== "control") return json(c.res, 400, { error: "only swarm controls are available here" });+ return relayCore(c.res, "POST", `/v1/sessions/${encodeURIComponent(c.params.id!)}/swarm`, JSON.stringify(body));+ },+ },{method: "GET",path: "/api/sessions/:id/approvals",diff --git a/plugins/web-ui/src/chat.ts b/plugins/web-ui/src/chat.tsindex 6e0f5f228..9793e57a1 100644--- a/plugins/web-ui/src/chat.ts+++ b/plugins/web-ui/src/chat.ts@@ -197,6 +197,7 @@ import {type PeekLine,type SubagentRow,} from "./subagent-activity";+import { createSwarmStrip } from "./swarm-strip.ts";import { newChatDraftKey, saveDraft, storedDraft } from "./drafts";import { createForkOriginController, forkOriginView } from "./fork-origin";import { base64ToBytes } from "./paste-text";@@ -1532,7 +1533,7 @@ export function createChatSurface(busy: agent.state.isStreaming,showPrompts: !messages.length,toolbar: html`${goalStrip(agent)} ${ctx.composer.queuedStrip(agent)} ${subagentStrip()}- ${backgroundActivityStrip()}`,+ ${swarmUi.strip(chatState.sessionId)} ${backgroundActivityStrip()}`,composer: ctx.composer.composerForm(agent),onPrompt: (prompt) => ctx.composer.fillSuggestedPrompt(prompt, agent),onDragEnter: (event) => ctx.composer.onDragEnter(event),@@ -1577,8 +1578,9 @@ export function createChatSurface(</div></section><div class="chat-bottom-dock">- ${goalStrip(agent)} ${ctx.composer.queuedStrip(agent)} ${subagentStrip()} ${backgroundActivityStrip()}- ${ctx.composer.composerForm(agent)} ${ctx.pane ? nothing : suggestions}+ ${goalStrip(agent)} ${ctx.composer.queuedStrip(agent)} ${subagentStrip()}+ ${swarmUi.strip(chatState.sessionId)} ${backgroundActivityStrip()} ${ctx.composer.composerForm(agent)}+ ${ctx.pane ? nothing : suggestions}</div></div>`;@@ -2444,6 +2446,7 @@ export function createChatSurface(`;}+ const swarmUi = createSwarmStrip(() => drawActiveChat());const SUBAGENT_ACK_KEY = "qm.subagentAck";const subagentUi = {expanded: false,diff --git a/plugins/web-ui/src/shell.css b/plugins/web-ui/src/shell.cssindex 9d5567cfc..d70bc17f7 100644--- a/plugins/web-ui/src/shell.css+++ b/plugins/web-ui/src/shell.css@@ -3413,6 +3413,11 @@ qm-markdown .tok-in {max-height: 42vh;overflow-y: auto;}+.swarm-bulk {+ display: flex;+ gap: 6px;+}+.bg-panel-note {padding: 6px 8px;font-size: 12px;diff --git a/plugins/web-ui/src/swarm-strip.ts b/plugins/web-ui/src/swarm-strip.tsnew file mode 100644index 000000000..337b1b3bb--- /dev/null+++ b/plugins/web-ui/src/swarm-strip.ts@@ -0,0 +1,235 @@+import { html, nothing, type TemplateResult } from "lit";+import { Users, ChevronRight } from "lucide";+import { api } from "./core-bridge.ts";+import { icon } from "./ui.ts";++interface SwarmPeer {+ id: string;+ parentId?: string;+ sessionId?: string;+ sessionUrl?: string;+ state: "reserved" | "ready" | "failed";+ control?: "paused" | "stopped";+ depth: number;+}++interface SwarmView {+ id: string;+ self: SwarmPeer;+ peers: SwarmPeer[];+ board?: { sandboxId: string };+ expiresAt: number;+}++type Control = "active" | "paused" | "stopped";+type Status = Control | "starting" | "failed";++const PAGE = 20;+const REFRESH_MS = 4_000;+const BULK: Record<Control, { label: string; from: (peer: SwarmPeer) => boolean }> = {+ paused: { label: "Pause all", from: (peer) => !peer.control },+ active: { label: "Resume all", from: (peer) => peer.control === "paused" },+ stopped: { label: "Stop all", from: (peer) => peer.control !== "stopped" },+};++function effective(peer: SwarmPeer, inherited: Status): Status {+ if (peer.control === "stopped" || inherited === "stopped") return "stopped";+ if (peer.state === "failed") return "failed";+ if (peer.control === "paused" || inherited === "paused") return "paused";+ return peer.state === "reserved" ? "starting" : "active";+}++function statuses(view: SwarmView): Map<string, Status> {+ const byId = new Map(view.peers.map((peer) => [peer.id, peer]));+ const result = new Map<string, Status>();+ const resolve = (peer: SwarmPeer, seen = new Set<string>()): Status => {+ const known = result.get(peer.id);+ if (known) return known;+ const parent = peer.parentId && !seen.has(peer.parentId) ? byId.get(peer.parentId) : undefined;+ const inherited = parent ? resolve(parent, seen.add(peer.id)) : "active";+ const status = effective(peer, inherited);+ result.set(peer.id, status);+ return status;+ };+ for (const peer of view.peers) resolve(peer);+ return result;+}++export function createSwarmStrip(redraw: () => void) {+ const ui = {+ sessionId: "",+ view: null as SwarmView | null,+ open: false,+ busy: "",+ error: "",+ shown: new Map<string, number>(),+ timer: undefined as ReturnType<typeof setTimeout> | undefined,+ };+ const schedule = (sessionId: string, active: boolean): void => {+ clearTimeout(ui.timer);+ ui.timer = undefined;+ if (!active || ui.sessionId !== sessionId) return;+ ui.timer = setTimeout(() => {+ if (!document.hidden) return void load(sessionId);+ document.addEventListener("visibilitychange", () => void load(sessionId), { once: true });+ }, REFRESH_MS);+ };+ const load = async (sessionId: string): Promise<void> => {+ const view = await api<SwarmView>(`/api/sessions/${encodeURIComponent(sessionId)}/swarm`).catch(() => null);+ if (ui.sessionId !== sessionId) return;+ ui.view = Array.isArray(view?.peers) ? view : null;+ const status = ui.view ? statuses(ui.view) : new Map();+ schedule(+ sessionId,+ [...status.entries()].some(([id, s]) => id !== ui.view?.id && (s === "active" || s === "starting")),+ );+ redraw();+ };+ const post = (sessionId: string, memberId: string, state: Control) =>+ api(`/api/sessions/${encodeURIComponent(sessionId)}/swarm`, {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({ action: "control", memberId, state }),+ });+ const control = async (key: string, memberIds: string[], state: Control): Promise<void> => {+ const sessionId = ui.sessionId;+ Object.assign(ui, { busy: key, error: "" });+ redraw();+ const results = await Promise.allSettled(memberIds.map((id) => post(sessionId, id, state)));+ const failed = results.filter((result) => result.status === "rejected").length;+ if (failed) ui.error = `Could not update ${failed} of ${memberIds.length} workers.`;+ ui.busy = "";+ await load(sessionId);+ };+ const bulk = (view: SwarmView, state: Control): void => {+ const targets = view.peers.filter((peer) => peer.parentId === view.id && BULK[state].from(peer)).map((p) => p.id);+ if (!targets.length) return;+ if (state === "stopped" && !confirm(`Stop all ${targets.length} worker groups? Stopped workers cannot be resumed.`))+ return;+ void control("bulk", targets, state);+ };+ const row = (peer: SwarmPeer, status: Status, view: SwarmView): TemplateResult => {+ const controllable = peer.depth > 0 && status !== "stopped" && peer.id !== view.self.id;+ const busy = ui.busy !== "";+ const name = peer.depth === 0 ? "Root session" : `Worker ${peer.id.slice(0, 8)}`;+ const label = peer.sessionUrl ? html`<a href=${peer.sessionUrl}>${name}</a>` : name;+ const next: Control = peer.control === "paused" ? "active" : "paused";+ return html`<div class="bg-row swarm-peer" style="padding-left:${Math.min(peer.depth, 6) * 12}px">+ <div class="bg-row-head static">+ <span class="bg-row-cmd">${label}</span>+ <span class="bg-row-meta">${status}</span>+ ${+ controllable+ ? html`${+ peer.control === "paused" || !peer.control+ ? html`<button+ type="button"+ ?disabled=${busy}+ @click=${() => void control(peer.id, [peer.id], next)}+ >+ ${next === "active" ? "Resume" : "Pause"}+ </button>`+ : nothing+ }+ <button type="button" ?disabled=${busy} @click=${() => void control(peer.id, [peer.id], "stopped")}>+ Stop+ </button>`+ : nothing+ }+ </div>+ </div>`;+ };+ const tree = (view: SwarmView, status: Map<string, Status>): TemplateResult[] => {+ const children = new Map<string, SwarmPeer[]>();+ for (const peer of view.peers) {+ if (!peer.parentId) continue;+ children.set(peer.parentId, [...(children.get(peer.parentId) ?? []), peer]);+ }+ const out: TemplateResult[] = [];+ const walk = (peer: SwarmPeer, seen: Set<string>): void => {+ if (seen.has(peer.id)) return;+ seen.add(peer.id);+ out.push(row(peer, status.get(peer.id) ?? "active", view));+ const kids = children.get(peer.id) ?? [];+ const limit = ui.shown.get(peer.id) ?? PAGE;+ for (const kid of kids.slice(0, limit)) walk(kid, seen);+ if (kids.length > limit)+ out.push(+ html`<button+ type="button"+ class="bg-panel-note swarm-more"+ style="padding-left:${Math.min(peer.depth + 1, 6) * 12}px"+ @click=${() => {+ ui.shown.set(peer.id, limit + PAGE);+ redraw();+ }}+ >+ Show ${Math.min(PAGE, kids.length - limit)} more of ${kids.length - limit} remaining+ </button>`,+ );+ };+ const root = view.peers.find((peer) => peer.id === view.id) ?? view.peers[0]!;+ walk(root, new Set());+ return out;+ };+ return {+ strip(sessionId: string | null): TemplateResult | typeof nothing {+ if (!sessionId) return nothing;+ if (sessionId !== ui.sessionId) {+ clearTimeout(ui.timer);+ Object.assign(ui, { sessionId, view: null, open: false, error: "", shown: new Map(), timer: undefined });+ void load(sessionId);+ }+ const view = ui.view;+ if (!view || view.peers.length < 2) return nothing;+ const status = statuses(view);+ const counts: Record<Status, number> = { active: 0, starting: 0, paused: 0, stopped: 0, failed: 0 };+ for (const peer of view.peers) if (peer.depth > 0) counts[status.get(peer.id) ?? "active"]++;+ const summary = (["active", "starting", "paused", "stopped", "failed"] as const)+ .filter((key) => key === "active" || counts[key])+ .map((key) => `${counts[key]} ${key}`)+ .join(" · ");+ const canBulk = view.self.depth === 0;+ return html`<section class="bg-activity swarm-activity ${ui.open ? "expanded" : ""}">+ <button+ type="button"+ class="bg-activity-strip"+ aria-expanded=${String(ui.open)}+ @click=${() => {+ ui.open = !ui.open;+ if (ui.open) void load(sessionId);+ redraw();+ }}+ >+ ${icon(Users, 13)}<span class="bg-activity-label">Swarm · ${summary}</span>+ <span class="bg-activity-toggle">${icon(ChevronRight, 14)}</span>+ </button>+ ${+ ui.open+ ? html`<div class="bg-panel" role="region" aria-label="Swarm workers">+ ${ui.error ? html`<div class="bg-panel-note">${ui.error}</div>` : nothing}+ ${+ view.board+ ? html`<div class="bg-panel-note">Shared board computer: ${view.board.sandboxId}</div>`+ : nothing+ }+ ${+ canBulk+ ? html`<div class="bg-panel-note swarm-bulk">+ ${(["paused", "active", "stopped"] as const).map(+ (state) =>+ html`<button type="button" ?disabled=${ui.busy !== ""} @click=${() => bulk(view, state)}>+ ${BULK[state].label}+ </button>`,+ )}+ </div>`+ : nothing+ }+ ${tree(view, status)}+ </div>`+ : nothing+ }+ </section>`;+ },+ };+}diff --git a/plugins/web-ui/test/pinned-composer-polish.test.ts b/plugins/web-ui/test/pinned-composer-polish.test.tsindex 3759e298b..9044b7378 100644--- a/plugins/web-ui/test/pinned-composer-polish.test.ts+++ b/plugins/web-ui/test/pinned-composer-polish.test.ts@@ -16,7 +16,7 @@ test("transcript top spacing and prompt gap stay compact at every density", () =test("background activity shares the queued inset above the composer", () => {assert.match(chat,- /ctx\.composer\.queuedStrip\(agent\)\} \$\{subagentStrip\(\)\} \$\{backgroundActivityStrip\(\)\}\s*\$\{ctx\.composer\.composerForm\(agent\)\}/,+ /ctx\.composer\.queuedStrip\(agent\)\} \$\{subagentStrip\(\)\}\s*\$\{swarmUi\.strip\(chatState\.sessionId\)\} \$\{backgroundActivityStrip\(\)\}\s*\$\{ctx\.composer\.composerForm\(agent\)\}/,);assert.match(css, /\.queued-strip,\s*\.chat-bottom-dock > \.bg-activity \{/);assert.doesNotMatch(css, /\.composer-wrap > \.bg-activity/);diff --git a/plugins/web-ui/test/swarm-server-route.test.ts b/plugins/web-ui/test/swarm-server-route.test.tsnew file mode 100644index 000000000..bc82d34ad--- /dev/null+++ b/plugins/web-ui/test/swarm-server-route.test.ts@@ -0,0 +1,74 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { createServer, type IncomingMessage } from "node:http";+import type { AddressInfo } from "node:net";+import { mintPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts";++const calls: { method: string; url: string; body: string; identity: boolean }[] = [];+const core = createServer((req: IncomingMessage, res) => {+ let raw = "";+ req.on("data", (chunk) => (raw += chunk));+ req.on("end", () => {+ calls.push({+ method: req.method ?? "GET",+ url: req.url ?? "",+ body: raw,+ identity: Boolean(req.headers[PORTAL_IDENTITY_HEADER.toLowerCase()]),+ });+ res.setHeader("content-type", "application/json");+ res.end(JSON.stringify({ ok: true }));+ });+});+await new Promise<void>((resolve) => core.listen(0, resolve));++process.env.CORE_API_URL = `http://localhost:${(core.address() as AddressInfo).port}`;+process.env.CORE_SIGNING_SECRET = "swarm-route-test";+process.env.WEB_UI_PRINCIPALS = "alice";++const { handler } = await import("../server/index.ts");+const surface = createServer((req, res) => void handler(req, res));+await new Promise<void>((resolve) => surface.listen(0, resolve));+const base = `http://localhost:${(surface.address() as AddressInfo).port}`;+const headers = {+ [PORTAL_IDENTITY_HEADER]: mintPortalIdentity({ p: "alice", exp: Date.now() + 60_000 }, "swarm-route-test"),+ "content-type": "application/json",+};++test.after(() => {+ surface.close();+ core.close();+});++test("swarm inspect and controls relay to core as the signed-in person", async () => {+ const before = calls.length;+ assert.equal((await fetch(`${base}/api/sessions/root-1/swarm`, { headers })).status, 200);+ const control = { action: "control", memberId: "worker-1", state: "paused" };+ const r = await fetch(`${base}/api/sessions/root-1/swarm`, {+ method: "POST",+ headers,+ body: JSON.stringify(control),+ });+ assert.equal(r.status, 200);+ const relayed = calls.slice(before).filter((call) => call.url.includes("/swarm"));+ assert.deepEqual(+ relayed.map(({ method, url, identity }) => ({ method, url: url.split("?")[0], identity })),+ [+ { method: "GET", url: "/v1/sessions/root-1/swarm", identity: true },+ { method: "POST", url: "/v1/sessions/root-1/swarm", identity: true },+ ],+ );+ assert.deepEqual(JSON.parse(relayed[1]!.body), control);+});++test("the web proxy exposes only swarm controls, not spawn or send", async () => {+ const before = calls.length;+ for (const action of ["spawn", "send", "context"]) {+ const r = await fetch(`${base}/api/sessions/root-1/swarm`, {+ method: "POST",+ headers,+ body: JSON.stringify({ action, requestId: "r", text: "hi" }),+ });+ assert.equal(r.status, 400);+ }+ assert.equal(calls.slice(before).filter((call) => call.url.includes("/swarm")).length, 0);+});diff --git a/src/api/agent-api-catalog.ts b/src/api/agent-api-catalog.tsindex cd2ed3056..0f8eac252 100644--- a/src/api/agent-api-catalog.ts+++ b/src/api/agent-api-catalog.ts@@ -63,7 +63,7 @@ const FAMILIES: AgentApiFamily[] = [match: (method, path) => path === "/v1/swarm" && (method === "GET" || method === "POST"),when: (view) => view.swarmsEnabled,guidance:- "Swarm workers are ordinary sessions with private blank computers. Inspect peers and their context, then send to chosen IDs or all; shared history is visible to every member. Notifications queue unattended turns. An optional forumSandboxId names an existing shared computer, selected explicitly per command with execute's sandbox_id.",+ "Swarm workers are ordinary sessions with private blank computers. Inspect peers and their context, then send to chosen IDs or all; shared history is visible to every member. Notifications queue unattended turns. Each swarm gets one shared board computer (inspect returns board.sandboxId) that the root and every worker reach with execute's sandbox_id; pass forumSandboxId at creation to use an existing shared computer instead. POST action control with memberId and state active, paused or stopped pauses delivery to, resumes, or stops a descendant worker.",routes: [{method: "GET",@@ -75,7 +75,7 @@ const FAMILIES: AgentApiFamily[] = [method: "POST",path: "/v1/swarm",summary:- "{action:'spawn',requestId,text,count?,context?,contexts?,forumSandboxId?,settings?,backend?} spawns one or an initial pool; {action:'context',context} updates own JSON; {action:'send',requestId,text,audience,replyTo?,notify?} sends to explicit peer ids or all. Retry the same requestId and payload for idempotency.",+ "{action:'spawn',requestId,text,count?,context?,contexts?,forumSandboxId?,settings?,backend?} spawns one or an initial pool; {action:'context',context} updates own JSON; {action:'send',requestId,text,audience,replyTo?,notify?} sends to explicit peer ids or all; {action:'control',memberId,state:'active'|'paused'|'stopped'} controls a descendant worker. Retry the same requestId and payload for idempotency.",},],},diff --git a/src/api/routes/surface.ts b/src/api/routes/surface.tsindex 4a95d313c..26bfa7be8 100644--- a/src/api/routes/surface.ts+++ b/src/api/routes/surface.ts@@ -680,7 +680,7 @@ async function listAgentApis(ctx: ApiCtx): Promise<void> {renderAgentApis(capability,{ isAdmin: admin.isAdmin, ...(admin.role ? { role: admin.role } : {}) },- { swarmsEnabled: Boolean(ctx.app.swarms) },+ { swarmsEnabled: Boolean(await ctx.app.swarms?.enabledFor(capability.actorId)) },),);}diff --git a/src/api/routes/swarms.ts b/src/api/routes/swarms.tsindex a508210df..b6406c5b6 100644--- a/src/api/routes/swarms.ts+++ b/src/api/routes/swarms.ts@@ -4,6 +4,8 @@ import type { SwarmCaller } from "../../swarms/swarm-service.ts";import { isObj } from "./shared.ts";import type { ApiCtx, Route } from "./route.ts";+const ACTION_FIELDS: Record<string, string[]> = { context: ["context"], control: ["memberId", "state"] };+async function swarmRequest(ctx: ApiCtx): Promise<void> {const { app, res, body, capability, actor, params, method, url } = ctx;if (!app.swarms) return sendJson(res, 503, { error: "swarm service unavailable" });@@ -35,7 +37,7 @@ async function swarmRequest(ctx: ApiCtx): Promise<void> {const allowed = new Set(["action",...(caller.kind === "human" ? ["runId"] : []),- ...(body.action === "context" ? ["context"] : ["requestId", "text"]),+ ...(ACTION_FIELDS[String(body.action)] ?? ["requestId", "text"]),...(body.action === "spawn" ? ["count", "context", "contexts", "forumSandboxId"] : []),...(body.action === "send" ? ["audience", "replyTo", "notify"] : []),...(body.action === "spawn" ? ["settings", "backend"] : []),@@ -45,6 +47,12 @@ async function swarmRequest(ctx: ApiCtx): Promise<void> {if (!("context" in body)) throw new Error("context required");return sendJson(res, 200, await app.swarms.context(caller, body.context));}+ if (body.action === "control") {+ if (typeof body.memberId !== "string" || !["active", "paused", "stopped"].includes(body.state as string))+ throw new Error("memberId and state (active, paused or stopped) required");+ const state = body.state as "active" | "paused" | "stopped";+ return sendJson(res, 200, { member: await app.swarms.control(caller, { memberId: body.memberId, state }) });+ }if (typeof body.requestId !== "string" || typeof body.text !== "string")throw new Error("requestId and text required");if (body.action === "spawn") {diff --git a/src/feature-flags.ts b/src/feature-flags.tsindex a1b3fed1a..2ebaf84d5 100644--- a/src/feature-flags.ts+++ b/src/feature-flags.ts@@ -9,6 +9,7 @@ export const FEATURE_NAMES = ["slack_loading_indicator","external_app_sharing","loop_triage",+ "swarms",] as const;export type FeatureName = (typeof FEATURE_NAMES)[number];diff --git a/src/swarms/swarm-service.ts b/src/swarms/swarm-service.tsindex 2d8ca05d6..5a784136c 100644--- a/src/swarms/swarm-service.ts+++ b/src/swarms/swarm-service.ts@@ -5,6 +5,7 @@ import type { CapabilityClaims } from "../auth/capability-token.ts";import type { OrchestratorInput } from "../core/orchestrator.ts";import type { AdvisoryLock } from "../persistence/advisory-lock.ts";import type { RunStore } from "../runs/run-store.ts";+import type { RunSignalStore } from "../runs/run-signal-store.ts";import type { SandboxResources } from "../sandbox/sandbox-resources.ts";import type { SessionStore } from "../sessions/session-store.ts";import { conversationScope } from "../resolution/resolution-service.ts";@@ -66,6 +67,7 @@ export interface SwarmService {id: string;self: SwarmMember;peers: SwarmMember[];+ board?: Swarm["board"];backend: Swarm["backend"];settings: SwarmSettings;expiresAt: number;@@ -74,6 +76,8 @@ export interface SwarmService {spawn(caller: SwarmCaller, input: SpawnInput): Promise<SwarmMember[]>;send(caller: SwarmCaller, input: MessageInput): Promise<SwarmMessage>;read(caller: SwarmCaller, options: { after?: number; replyTo?: string; waitMs?: number }): Promise<SwarmMessage[]>;+ control(caller: SwarmCaller, input: { memberId: string; state: SwarmControlState }): Promise<SwarmMember>;+ enabledFor(actorId: string): Promise<boolean>;binding(input: OrchestratorInput): Promise<{ sandboxId?: string; rootSessionId: string; member: SwarmMember } | null>;}@@ -105,6 +109,32 @@ function jsonContext(value: unknown, max: number): unknown {return JSON.parse(encoded) as unknown;}+type SwarmControlState = "active" | "paused" | "stopped";++const memberIndex = new WeakMap<SwarmMember[], Map<string, SwarmMember>>();++function memberById(swarm: Swarm, id: string): SwarmMember | undefined {+ let index = memberIndex.get(swarm.members);+ if (!index || index.size !== swarm.members.length) {+ index = new Map(swarm.members.map((member) => [member.id, member]));+ memberIndex.set(swarm.members, index);+ }+ return index.get(id);+}++function controlState(swarm: Swarm, memberId: string): SwarmControlState {+ let state: SwarmControlState = "active";+ const seen = new Set<string>();+ for (let id: string | undefined = memberId; id && !seen.has(id);) {+ seen.add(id);+ const member = memberById(swarm, id);+ if (member?.control === "stopped") return "stopped";+ if (member?.control === "paused") state = "paused";+ id = member?.parentId;+ }+ return state;+}+function signature(value: unknown): string {return createHash("sha256").update(canonicalJson(value)).digest("hex");}@@ -168,8 +198,11 @@ export function createSwarmService(deps: {sandboxes: SandboxResources;lock: AdvisoryLock;authorize(claims: Pick<CapabilityClaims, "actorId" | "scopeId" | "scopeVersion" | "members">): Promise<boolean>;+ enabled?(actorId: string): Promise<boolean>;+ signals?: Pick<RunSignalStore, "send">;defaults?: SwarmSettings;}): SwarmService {+ const enabledFor = deps.enabled ?? (async () => true);const { store, sessions, runs } = deps;const update = (auth: Authority, mutate: (swarm: Swarm) => void) => store.update(auth.rootId, mutate, auth.fence);const defaults = resolveSwarmSettings(deps.defaults);@@ -248,6 +281,7 @@ export function createSwarmService(deps: {auth: Authority,settings: SwarmSettings,backend?: string,+ forumSandboxId?: string,): Promise<Swarm> {const runId = caller.kind === "agent" ? caller.claims.runId : caller.runId;const run = runId ? await runs.get(runId) : null;@@ -300,6 +334,7 @@ export function createSwarmService(deps: {backend: provider.name,createdAt,expiresAt: createdAt + settings.lifetimeMs,+ ...(forumSandboxId ? {} : { board: { sandboxId: randomUUID() } }),members: [{id: session.id,@@ -334,9 +369,11 @@ export function createSwarmService(deps: {for (const message of swarm.messages) {for (const [recipientId, notification] of Object.entries(message.notifications)) {if (notification.state !== "pending") continue;- const recipient = swarm.members.find((member) => member.id === recipientId);+ const recipient = memberById(swarm, recipientId);if (!recipient || recipient.state === "reserved") continue;- if (recipient.state === "failed" || Date.now() >= swarm.expiresAt) {+ const control = controlState(swarm, recipient.id);+ if (control === "paused") continue;+ if (recipient.state === "failed" || control === "stopped" || Date.now() >= swarm.expiresAt) {await step(() =>store.update(swarm.id, (current) => {current.messages.find((item) => item.id === message.id)!.notifications[recipientId] = { state: "failed" };@@ -386,7 +423,7 @@ export function createSwarmService(deps: {};try {let swarm = await step(() => store.get(rootId));- if (!swarm) return;+ if (!swarm || !(await step(() => enabledFor(swarm!.ownerId)))) return;if (phase === "delivery") {await deliver(swarm, step);return;@@ -401,18 +438,24 @@ export function createSwarmService(deps: {}),);if (!member.sandboxId) throw new Error("missing sandbox reservation");+ const provision = (name: string, id: string) => {+ const provisionDeadline = Math.min(deadline, Date.now() + SWARM_LIMITS.provisionMs);+ return step(+ () => deps.sandboxes.create(swarm!.ownerId, swarm!.scopeId, swarm!.backend, name, id),+ SWARM_LIMITS.provisionMs,+ ).catch((error: unknown) => {+ provisioningTimedOut = Date.now() >= provisionDeadline;+ throw error;+ });+ };if (member.forumSandboxId) {- const forum = await step(() => deps.sandboxes.access(swarm!.ownerId, member.forumSandboxId!));+ const forum =+ member.forumSandboxId === swarm.board?.sandboxId+ ? await provision("Swarm board", member.forumSandboxId)+ : await step(() => deps.sandboxes.access(swarm!.ownerId, member.forumSandboxId!));if (forum.ownerScopeId !== swarm.scopeId) throw new Error("forum scope mismatch");}- const provisionDeadline = Math.min(deadline, Date.now() + SWARM_LIMITS.provisionMs);- await step(- () => deps.sandboxes.create(swarm!.ownerId, swarm!.scopeId, swarm!.backend, "Swarm worker", member.id),- SWARM_LIMITS.provisionMs,- ).catch((error: unknown) => {- provisioningTimedOut = Date.now() >= provisionDeadline;- throw error;- });+ await provision("Swarm worker", member.id);const session = await step(() =>sessions.getOrCreateByThread(member.threadRef,@@ -534,6 +577,7 @@ export function createSwarmService(deps: {id: swarm.id,self: view(self),peers: swarm.members.map(view),+ ...(swarm.board ? { board: swarm.board } : {}),backend: swarm.backend,settings: swarm.settings,expiresAt: swarm.expiresAt,@@ -606,7 +650,9 @@ export function createSwarmService(deps: {depth: parent.depth + 1,context,sandboxId: id,- ...(input.forumSandboxId ? { forumSandboxId: input.forumSandboxId } : {}),+ ...((input.forumSandboxId ?? swarm.board?.sandboxId)+ ? { forumSandboxId: input.forumSandboxId ?? swarm.board!.sandboxId }+ : {}),state: "reserved",attempts: 0,};@@ -630,7 +676,7 @@ export function createSwarmService(deps: {let updated: Swarm;if (existing) updated = await update(auth, reserve);else {- const initial = await prepareInitialSwarm(caller, auth, settings, input.backend);+ const initial = await prepareInitialSwarm(caller, auth, settings, input.backend, input.forumSandboxId);reserve(initial);updated = await store.create(initial, auth.fence);if (!updated.spawnRequests[key]) updated = await update(auth, reserve);@@ -652,14 +698,17 @@ export function createSwarmService(deps: {if (previous.signature !== fingerprint) throw new Error("requestId reused with different content");return swarm.messages.find((message) => message.id === previous.messageId)!;}- const eligible: SwarmMember[] = [];- for (const member of swarm.members) {- if (member.state !== "ready" || !member.sessionId) continue;- const session = await sessions.get(member.sessionId);- const participants = await sessions.participantsOf(member.sessionId);- if (session?.scopeId === swarm.scopeId && rosterMatches(participants, swarm.participants))- eligible.push(member);- }+ const checked = await Promise.all(+ swarm.members.map(async (member) => {+ if (member.state !== "ready" || !member.sessionId) return false;+ const [session, participants] = await Promise.all([+ sessions.get(member.sessionId),+ sessions.participantsOf(member.sessionId),+ ]);+ return session?.scopeId === swarm.scopeId && rosterMatches(participants, swarm.participants);+ }),+ );+ const eligible = swarm.members.filter((_, index) => checked[index]);const audience = resolveAudience(input.audience, eligible);const id = randomUUID();const updated = await update(auth, (swarm) => {@@ -715,6 +764,42 @@ export function createSwarmService(deps: {({ swarm } = await load(caller));}},+ enabledFor,+ async control(caller, input) {+ const { auth, swarm } = await load(caller);+ if (!["active", "paused", "stopped"].includes(input.state)) throw new Error("invalid control state");+ const target = swarm.members.find((member) => member.id === input.memberId);+ if (!target) throw new Error("unknown swarm member");+ if (target.id === swarm.id) throw new Error("the root session cannot be controlled");+ if (caller.kind === "agent") {+ let ancestor: string | undefined = target.id;+ while (ancestor && ancestor !== auth.memberId)+ ancestor = swarm.members.find((member) => member.id === ancestor)?.parentId;+ if (!ancestor) throw new Error("agents may only control themselves or their descendants");+ }+ const updated = await update(auth, (current) => {+ const member = current.members.find((peer) => peer.id === target.id)!;+ if (member.control === "stopped") throw new Error("stopped workers cannot be resumed");+ if (input.state === "active") delete member.control;+ else member.control = input.state;+ });+ if (input.state === "stopped") {+ const stopped = new Set(+ updated.members.filter((m) => controlState(updated, m.id) === "stopped").map((m) => m.id),+ );+ const runIds = updated.messages.flatMap((message) =>+ Object.entries(message.notifications).flatMap(([id, n]) => (stopped.has(id) && n.runId ? [n.runId] : [])),+ );+ await Promise.all(+ runIds.map(async (runId) => {+ if (await runs.withdraw(runId, { unstartedOnly: true }).catch(() => false)) return;+ const run = await runs.get(runId);+ if (run?.status === "running") await deps.signals?.send(runId, { kind: "abort" }).catch(() => false);+ }),+ );+ }+ return view(updated.members.find((member) => member.id === target.id)!);+ },async binding(input) {const session = await sessions.getByThread(input.conversation.threadRef);if (!session) {@@ -732,6 +817,7 @@ export function createSwarmService(deps: {const member = swarm.members.find((peer) => peer.id === identity.memberId);if (!member || member.state !== "ready" || member.sessionId !== session.id || session.scopeId !== swarm.scopeId)throw new NonRetryableTurnError("invalid swarm membership");+ if (controlState(swarm, member.id) === "stopped") throw new NonRetryableTurnError("swarm worker stopped");if (!input.swarm) {if (!(await sessions.participantsOf(session.id)).includes(input.actor.id) ||diff --git a/src/swarms/swarm-settings.ts b/src/swarms/swarm-settings.tsindex 4afb6b381..17440768c 100644--- a/src/swarms/swarm-settings.ts+++ b/src/swarms/swarm-settings.ts@@ -14,7 +14,7 @@ export const SWARM_DEFAULTS = {export type SwarmSettings = typeof SWARM_DEFAULTS;export const SWARM_MAXIMUMS: Readonly<SwarmSettings> = {- agents: 64,+ agents: 256,depth: 8,messages: 256,notifications: 1_024,diff --git a/src/swarms/swarm-store.ts b/src/swarms/swarm-store.tsindex 9756eac15..7ed0b3428 100644--- a/src/swarms/swarm-store.ts+++ b/src/swarms/swarm-store.ts@@ -27,6 +27,7 @@ export interface SwarmMember {sandboxId?: string;forumSandboxId?: string;state: "reserved" | "ready" | "failed";+ control?: "paused" | "stopped";attempts: number;cleanupPending?: boolean;error?: string;@@ -56,6 +57,7 @@ export interface Swarm {template: OrchestratorInput;settings: SwarmSettings;backend: SandboxBackendName;+ board?: { sandboxId: string };members: SwarmMember[];messages: SwarmMessage[];spawnRequests: Record<string, { memberIds: string[]; signature: string }>;diff --git a/src/wiring.ts b/src/wiring.tsindex a59c27110..2b8af52e2 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -1548,7 +1548,10 @@ export function buildApp(runs,sandboxes: sandboxResources,lock: advisoryLock,+ signals: runSignals,+ enabled: (actorId) => featureFlags.enabled("swarms", scopeId("personal", actorId)),authorize: async (claims) => {+ if (!(await featureFlags.enabled("swarms", scopeId("personal", claims.actorId)))) return false;await identity.refresh();return (identity.isInternal(identity.classify(claims.actorId)) &&diff --git a/test/swarm-board-controls.test.ts b/test/swarm-board-controls.test.tsnew file mode 100644index 000000000..59c64d1b4--- /dev/null+++ b/test/swarm-board-controls.test.ts@@ -0,0 +1,100 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { createSwarmService } from "../src/swarms/swarm-service.ts";+import { swarmFixture } from "./support/swarm-fixture.ts";++test("a swarm provisions one shared board computer that the root and every worker share", async () => {+ const fixture = await swarmFixture();+ const workers = await fixture.service.spawn(fixture.caller, { requestId: "pool", text: "Work", count: 2 });+ await fixture.service.sweep();+ const swarm = (await fixture.store.get(fixture.root.id))!;+ const board = (await fixture.records.get(swarm.board!.sandboxId))!;+ assert.equal(board.ownerScopeId, fixture.root.scopeId);+ assert.equal(board.name, "Swarm board");+ for (const worker of workers)+ assert.equal(swarm.members.find((m) => m.id === worker.id)!.forumSandboxId, swarm.board!.sandboxId);+ const inspected = await fixture.service.inspect(fixture.caller);+ assert.equal(inspected.board?.sandboxId, swarm.board!.sandboxId);+ const worker = await fixture.service.inspect(await fixture.workerCaller(workers[0]!.id));+ assert.equal(worker.self.forumSandboxId, swarm.board!.sandboxId);+});++test("an explicit forum computer replaces the automatic board", async () => {+ const fixture = await swarmFixture();+ const forum = await fixture.sandboxes.create("alice", fixture.root.scopeId as never, "modal", "Forum");+ await fixture.service.spawn(fixture.caller, { requestId: "one", text: "Work", forumSandboxId: forum.id });+ await fixture.service.sweep();+ const swarm = (await fixture.store.get(fixture.root.id))!;+ assert.equal(swarm.board, undefined);+ assert.equal(swarm.members[1]!.forumSandboxId, forum.id);+});++test("pausing holds delivery, resuming releases it, and stopping is terminal", async () => {+ const fixture = await swarmFixture();+ const [worker] = await fixture.service.spawn(fixture.caller, { requestId: "one", text: "Work" });+ await fixture.service.control(fixture.caller, { memberId: worker!.id, state: "paused" });+ await fixture.service.sweep();+ let swarm = (await fixture.store.get(fixture.root.id))!;+ assert.equal(swarm.members[1]!.state, "ready");+ assert.equal(swarm.messages[0]!.notifications[worker!.id]!.state, "pending");+ assert.equal((await fixture.service.inspect(fixture.caller)).peers[1]!.control, "paused");+ await fixture.service.control(fixture.caller, { memberId: worker!.id, state: "active" });+ await fixture.service.sweep();+ swarm = (await fixture.store.get(fixture.root.id))!;+ const notification = swarm.messages[0]!.notifications[worker!.id]!;+ assert.equal(notification.state, "queued");+ await fixture.service.control(fixture.caller, { memberId: worker!.id, state: "stopped" });+ assert.equal(await fixture.runs.get(notification.runId!), null);+ await assert.rejects(+ fixture.service.control(fixture.caller, { memberId: worker!.id, state: "active" }),+ /cannot be resumed/,+ );+ await assert.rejects(+ fixture.service.control(fixture.caller, { memberId: fixture.root.id, state: "stopped" }),+ /root session/,+ );+});++test("a worker cannot control a sibling", async () => {+ const fixture = await swarmFixture();+ const [first, second] = await fixture.service.spawn(fixture.caller, { requestId: "pool", text: "Work", count: 2 });+ await fixture.service.sweep();+ await assert.rejects(+ fixture.service.control(await fixture.workerCaller(first!.id), { memberId: second!.id, state: "stopped" }),+ /descendants/,+ );+});++test("a swarm owner without the flag gets no reconciliation", async () => {+ const fixture = await swarmFixture();+ let enabled = true;+ const service = createSwarmService({ ...fixture.serviceOptions, enabled: async () => enabled });+ await service.spawn(fixture.caller, { requestId: "one", text: "Work" });+ enabled = false;+ await service.sweep();+ const swarm = (await fixture.store.get(fixture.root.id))!;+ assert.equal(swarm.members[1]!.state, "reserved");+ assert.equal((await fixture.records.get(swarm.board!.sandboxId)) ?? null, null);+ assert.equal(await service.enabledFor("alice"), false);+});++test("pausing and stopping the root's direct children cascades across a 120-worker swarm", async () => {+ const fixture = await swarmFixture();+ const workers = await fixture.service.spawn(fixture.caller, {+ requestId: "wide",+ text: "Work",+ count: 120,+ settings: { agents: 128 },+ });+ assert.equal(workers.length, 120);+ const children = workers.filter((worker) => worker.parentId === fixture.root.id);+ await Promise.all(children.map((w) => fixture.service.control(fixture.caller, { memberId: w.id, state: "paused" })));+ await fixture.service.sweep();+ let swarm = (await fixture.store.get(fixture.root.id))!;+ assert.ok(workers.every((w) => swarm.messages[0]!.notifications[w.id]?.state !== "queued"));+ await Promise.all(children.map((w) => fixture.service.control(fixture.caller, { memberId: w.id, state: "stopped" })));+ await fixture.service.sweep();+ swarm = (await fixture.store.get(fixture.root.id))!;+ assert.ok(workers.every((w) => swarm.messages[0]!.notifications[w.id]?.state !== "queued"));+ assert.equal((await fixture.service.inspect(fixture.caller)).peers.length, 121);+});diff --git a/test/swarm-modal.test.ts b/test/swarm-modal.test.tsindex 40b73a5d7..91784e387 100644--- a/test/swarm-modal.test.ts+++ b/test/swarm-modal.test.ts@@ -66,7 +66,7 @@ test("real Modal adapter creates blank worker computers without snapshot or restassert.equal(snapshotCalls, 0);assert.equal(restoreCalls, 0);assert.equal(snapshotStore.puts(), 0);- assert.equal(fake.totalCreated(), 3);+ assert.equal(fake.totalCreated(), 4);} finally {fake.cleanup();}diff --git a/test/swarm-orchestrator.test.ts b/test/swarm-orchestrator.test.tsindex d267a318f..aaa5f0c2c 100644--- a/test/swarm-orchestrator.test.ts+++ b/test/swarm-orchestrator.test.ts@@ -47,7 +47,13 @@ mock.module("../src/harness/mock-harness.ts", {},},});-const { buildApp } = await import("../src/wiring.ts");+const wiring = await import("../src/wiring.ts");+const { orgId } = await import("../src/config.ts");+const buildApp = (...args: Parameters<typeof wiring.buildApp>): ReturnType<typeof wiring.buildApp> => {+ const built = wiring.buildApp(...args);+ void built.featureFlags.setEnabled("swarms", `org:${orgId()}`, true, "test");+ return built;+};test.after(() => fake.cleanup());for (const kind of ["command", "security-screen"] as const) {@@ -573,3 +579,16 @@ test("disabled swarms park queued notifications once without running the model oawait built.runtime.stop();}});++test("swarms stay off for a person until the swarms flag names their personal scope", async () => {+ const built = wiring.buildApp(testConfig());+ try {+ assert.ok(built.app.swarms);+ assert.equal(await built.app.swarms.enabledFor("U1"), false);+ await built.featureFlags.setEnabled("swarms", "personal:U1", true, "test");+ assert.equal(await built.app.swarms.enabledFor("U1"), true);+ assert.equal(await built.app.swarms.enabledFor("U2"), false);+ } finally {+ await built.runtime.stop();+ }+});diff --git a/test/swarms.test.ts b/test/swarms.test.tsindex cceddea17..e0d5f2c07 100644--- a/test/swarms.test.ts+++ b/test/swarms.test.ts@@ -246,7 +246,7 @@ test("timed-out provider calls retain bounded execution slots across repeated swfor (const release of releases.slice(1)) release();await new Promise<void>((resolve) => setImmediate(resolve));await first.service.sweep();- assert.equal(entered.length, fixtures.length * 3 - 2);+ assert.equal(entered.length, fixtures.length * 4 - SWARM_LIMITS.sweepConcurrency - 2);assert.equal(maximum, SWARM_LIMITS.sweepConcurrency);assert.equal(active, 1);} finally {@@ -679,8 +679,10 @@ test("restart between enqueue and outbox acknowledgement does not redeliver", as});test("sandbox reservation survives a crash after provisioning without creating another disk", async () => {- const { service, serviceOptions, caller, root, sandboxes, provisioned } = await swarmFixture();+ const { service, serviceOptions, caller, root, sandboxes, provisioned, store } = await swarmFixture();const [worker] = await service.spawn(caller, { requestId: "one", text: "Work" });+ const { board } = (await store.get(root.id))!;+ await sandboxes.create("alice", root.scopeId, "modal", "Swarm board", board!.sandboxId);await sandboxes.create("alice", root.scopeId, "modal", "Swarm worker", worker!.id);const count = provisioned.length;await createSwarmService(serviceOptions).sweep();