Multiplayer agent harness for work.
Allow deployments to disable swarms (#1347)
Regan Belle32e94315eae
diff --git a/docs/swarms.md b/docs/swarms.mdindex c84336221..003ff5059 100644--- a/docs/swarms.md+++ b/docs/swarms.md@@ -229,3 +229,9 @@ exercise the HTTP spawn/reply flow across an application restart with real durabstate. These tests use deterministic model and sandbox doubles; live provider andmodel acceptance remains a separate deployment check. The agent-board UI isintentionally deferred.++Operators can set `SWARMS_ENABLED=false` to disable the swarm service, API,+agent discovery, and background reconciliation. Existing swarm records and+computers are retained. Already queued swarm turns fail without model execution+and are not retried. The default is `true`. Restart core and workers after changing+this deployment setting.diff --git a/src/api/agent-api-catalog.ts b/src/api/agent-api-catalog.tsindex 1270e9b43..17a1ec900 100644--- a/src/api/agent-api-catalog.ts+++ b/src/api/agent-api-catalog.ts@@ -10,6 +10,7 @@ interface AgentApiRoute {interface AgentApiView {claims: CapabilityClaims;isAdmin: boolean;+ swarmsEnabled: boolean;}interface AgentApiFamily {@@ -35,6 +36,7 @@ const FAMILIES: AgentApiFamily[] = [},{match: (method, path) => path === "/v1/swarm" && (method === "GET" || method === "POST"),+ when: (view) => view.swarmsEnabled,guidance:"Swarm workers are ordinary sessions with private blank computers. Inspect peers and their context, then send to chosen IDs or all; shared history is visible to every member. Notifications queue unattended turns. An optional forumSandboxId names an existing shared computer, selected explicitly per command with execute's sandbox_id.",routes: [@@ -907,10 +909,15 @@ export interface AgentApiListing {guidance: string[];}-export function renderAgentApis(claims: CapabilityClaims, admin: { isAdmin: boolean; role?: string }): AgentApiListing {+export function renderAgentApis(+ claims: CapabilityClaims,+ admin: { isAdmin: boolean; role?: string },+ features: { swarmsEnabled: boolean },+): AgentApiListing {const view: AgentApiView = {claims,isAdmin: admin.isAdmin,+ swarmsEnabled: features.swarmsEnabled,};const visible = [...FAMILIES, WHOAMI_FOR_ALL].filter((f) => f.when?.(view) ?? true);return {diff --git a/src/api/routes/surface.ts b/src/api/routes/surface.tsindex b269bcbf3..28783544f 100644--- a/src/api/routes/surface.ts+++ b/src/api/routes/surface.ts@@ -678,7 +678,11 @@ async function listAgentApis(ctx: ApiCtx): Promise<void> {return sendJson(res,200,- renderAgentApis(capability, { isAdmin: admin.isAdmin, ...(admin.role ? { role: admin.role } : {}) }),+ renderAgentApis(+ capability,+ { isAdmin: admin.isAdmin, ...(admin.role ? { role: admin.role } : {}) },+ { swarmsEnabled: Boolean(ctx.app.swarms) },+ ),);}diff --git a/src/config.ts b/src/config.tsindex 1c22a37ad..9b96d3bac 100644--- a/src/config.ts+++ b/src/config.ts@@ -45,6 +45,7 @@ export interface Config {slackContextSource?: SlackContextSource;suggestedActivitiesEnabled?: boolean;suggestedActivitiesContext?: string;+ swarmsEnabled?: boolean;swarmDefaults?: SwarmSettings;production: boolean;allowUnauthenticatedCore: boolean;@@ -1376,6 +1377,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config {execTimeoutMaxMs:(numEnvStrict("EXEC_TIMEOUT_MAX_SEC", env.EXEC_TIMEOUT_MAX_SEC) ?? CONFIG_DEFAULTS.execTimeoutMaxSec) * 1000,turnWallClockMs,+ swarmsEnabled: boolEnvStrict("SWARMS_ENABLED", env.SWARMS_ENABLED) ?? true,swarmDefaults,runMaxAgeMs,runWaitMs: (turnWallClockMs > 0 ? turnWallClockMs : runMaxAgeMs) + 60_000,diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.tsindex 8d6b40a88..40bae9196 100644--- a/src/core/orchestrator.ts+++ b/src/core/orchestrator.ts@@ -523,7 +523,11 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {},async handleTurn(input: OrchestratorInput): Promise<TurnResult> {- if (input.swarm && !deps.swarms) throw new Error("swarm service unavailable");+ if (+ !deps.swarms &&+ (input.swarm || input.surface === "swarm" || input.conversation.threadRef.startsWith("swarm:"))+ )+ throw new NonRetryableTurnError("swarm service unavailable");const swarmBinding = await deps.swarms?.binding(input);const swarmEntryProvenance = input.swarm ? { origin: "automation", swarm: input.swarm } : {};await deps.refreshModels?.();diff --git a/src/delivery/run-result-delivery.ts b/src/delivery/run-result-delivery.tsindex d5ccc89db..67eeeacc9 100644--- a/src/delivery/run-result-delivery.ts+++ b/src/delivery/run-result-delivery.ts@@ -102,7 +102,7 @@ const FAILURE_RECORD_SCAN_LIMIT = 200;const FAILURE_RECORD_WAIT_MS = 10 * 60_000;export async function recordRunFailureEntry(sessions: TurnFailureSessions, run: Run): Promise<boolean> {- if (run.status !== "failed") return false;+ if (run.status !== "failed" || run.request.swarm) return false;const session = await sessions.getByThread(run.sessionId);if (!session) {console.error(diff --git a/src/wiring.ts b/src/wiring.tsindex 7a6f317c8..42d53ac56 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -1353,7 +1353,7 @@ export function buildApp(};const swarmStoreKind = config.databaseUrl ? "postgres" : "memory";const swarms =- config.sessionStore === swarmStoreKind && runStoreKind === swarmStoreKind+ config.swarmsEnabled !== false && config.sessionStore === swarmStoreKind && runStoreKind === swarmStoreKind? createSwarmService({defaults: config.swarmDefaults,store: createSwarmStore(artifactMap<SwarmStorage>("swarms"), {diff --git a/test/agent-api-discovery.test.ts b/test/agent-api-discovery.test.tsindex e3dfda62f..9ce3b7f9a 100644--- a/test/agent-api-discovery.test.ts+++ b/test/agent-api-discovery.test.ts@@ -16,11 +16,12 @@ import { testConfig } from "./support/test-config.ts";const SECRET = "discovery-test-secret".repeat(3);const ORG = scopeId("org", "default-org");-async function start() {+async function start(swarmsEnabled = true) {const built = buildApp(testConfig({dataDir: mkdtempSync(join(tmpdir(), "agent-apis-")),signingSecret: SECRET,+ swarmsEnabled,}),);const server = createServer(built.app, {@@ -242,3 +243,28 @@ test("discovery includes admin routes for a verified human thread reply", asyncawait s.close();}});++test("disabled swarms are absent from discovery and reject direct API calls", async () => {+ const { base, built, close } = await start(false);+ try {+ assert.equal(built.app.swarms, undefined);+ const token = await capFor("U1");+ const { status, body } = await listApis(base, token);+ assert.equal(status, 200);+ assert.ok(!paths(body).includes("/v1/swarm"));+ assert.ok(body.guidance.every((line: string) => !line.includes("Swarm")));+ for (const method of ["GET", "POST"]) {+ const response = await fetch(`${base}/v1/swarm`, {+ method,+ headers: { "x-agent-capability": token, "content-type": "application/json" },+ ...(method === "POST"+ ? { body: JSON.stringify({ action: "spawn", requestId: "disabled", text: "work" }) }+ : {}),+ });+ assert.equal(response.status, 503);+ assert.deepEqual(await response.json(), { error: "swarm service unavailable" });+ }+ } finally {+ await close();+ }+});diff --git a/test/run-result-delivery.test.ts b/test/run-result-delivery.test.tsindex 3751824d6..14ffa45cb 100644--- a/test/run-result-delivery.test.ts+++ b/test/run-result-delivery.test.ts@@ -2,7 +2,12 @@ import { test } from "node:test";import assert from "node:assert/strict";import { createMemoryRunStore } from "../src/runs/memory-run-store.ts";import { createDeliveryStore } from "../src/delivery/delivery-store.ts";-import { runResultDelivery, wireRunResultDeliveries } from "../src/delivery/run-result-delivery.ts";+import {+ runResultDelivery,+ wireRunResultDeliveries,+ recordRunFailureEntry,+ type TurnFailureSessions,+} from "../src/delivery/run-result-delivery.ts";import type { Run } from "../src/runs/run-store.ts";import type { OrchestratorInput } from "../src/core/orchestrator.ts";import type { Principal, TurnResult } from "../src/types.ts";@@ -371,3 +376,14 @@ test("private session turns cannot deliver even with a stale destination", () =>null,);});++test("failed internal swarm notifications never append user-facing failure entries", async () => {+ const failed = run({ status: "failed", result: { status: "failed", reason: "swarm service unavailable" } });+ failed.request.swarm = { swarmId: "root", recipientId: "root", messageId: "message" };+ const sessions = {+ getByThread: async () => {+ throw new Error("must not access the transcript");+ },+ } as unknown as TurnFailureSessions;+ assert.equal(await recordRunFailureEntry(sessions, failed), false);+});diff --git a/test/swarm-orchestrator.test.ts b/test/swarm-orchestrator.test.tsindex 49d602f2f..bb8fef2b4 100644--- a/test/swarm-orchestrator.test.ts+++ b/test/swarm-orchestrator.test.ts@@ -490,3 +490,66 @@ test("a resolved command approval informs the model without changing its requestawait built.runtime.stop();}});++test("disabled swarms park queued notifications once without running the model or writing failures", async () => {+ const built = buildApp(testConfig({ swarmsEnabled: false }));+ try {+ const rootTurn = await built.app.turn({+ surface: "test",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: "disabled-swarm-root" },+ origin: { kind: "human" },+ text: "Initialize",+ });+ assert.equal(rootTurn.status, "ok");+ const root = (await built.sessions.get(rootTurn.sessionId!))!;+ const rootRun = (await built.runs.list()).find((run) => run.request.conversation.threadRef === root.threadRef)!;+ let swarmModelCalls = 0;+ exerciseTurn = async (turn) => {+ if (turn.input.includes("disabled-swarm-message")) swarmModelCalls++;+ };+ const { run } = await built.runs.enqueue({+ sessionId: root.threadRef,+ request: {+ ...rootRun.request,+ surface: "swarm",+ turnWallClockMs: 600_000,+ origin: { kind: "automation" },+ text: "disabled-swarm-message",+ swarm: { swarmId: root.id, recipientId: root.id, messageId: "queued" },+ },+ });+ built.runtime.start();+ const completed = await built.runs.waitFor(run.id, 15_000);+ assert.equal(completed.status, "failed");+ assert.equal(completed.attempts, 1);+ assert.equal(completed.result?.reason, "swarm service unavailable");+ assert.equal(swarmModelCalls, 0);+ assert.equal(runResultDelivery(completed), null);+ const entries = await built.sessions.getEntries(root.id);+ assert.ok(!entries.some((entry) => JSON.stringify(entry.payload).includes("turn_failure")));+ for (const [surface, threadRef] of [+ ["swarm", "worker"],+ ["web", "swarm:root:worker"],+ ]) {+ const { run: manual } = await built.runs.enqueue({+ sessionId: threadRef!,+ request: {+ ...rootRun.request,+ surface: surface!,+ conversation: { ...rootRun.request.conversation, threadRef: threadRef! },+ origin: { kind: "human" },+ text: "disabled-swarm-message",+ },+ });+ const rejected = await built.runs.waitFor(manual.id, 15_000);+ assert.equal(rejected.status, "failed");+ assert.equal(rejected.attempts, 1);+ assert.equal(rejected.result?.reason, "swarm service unavailable");+ assert.equal(swarmModelCalls, 0);+ }+ } finally {+ exerciseTurn = undefined;+ await built.runtime.stop();+ }+});diff --git a/test/swarm-settings.test.ts b/test/swarm-settings.test.tsindex 40375059b..714c36f4f 100644--- a/test/swarm-settings.test.ts+++ b/test/swarm-settings.test.ts@@ -148,3 +148,12 @@ test("failed initial persistence leaves no configuration and a complete pool canassert.equal(swarm.messages.length, 1);assert.equal(swarm.settings.turnMs, 700_000);});++test("operators can disable swarms without changing other background work", () => {+ assert.equal(loadConfig({}).swarmsEnabled, true);+ assert.equal(loadConfig({ SWARMS_ENABLED: "true" }).swarmsEnabled, true);+ const disabled = loadConfig({ SWARMS_ENABLED: "false" });+ assert.equal(disabled.swarmsEnabled, false);+ assert.equal(disabled.backgroundWorkEnabled, true);+ assert.throws(() => loadConfig({ SWARMS_ENABLED: "invalid" }), /SWARMS_ENABLED/);+});