Skip to content

Agent Mesh

development/examples: add warm agent pool (pool-manager + reviewer pool)

Tomas Tormoef30fe0f8915
diff --git a/development/examples/code-reviewer-mcp/review_server.mjs b/development/examples/code-reviewer-mcp/review_server.mjs
index 3ea44f05..ef0486b0 100644
--- a/development/examples/code-reviewer-mcp/review_server.mjs
+++ b/development/examples/code-reviewer-mcp/review_server.mjs
@@ -10,6 +10,11 @@ const REVIEW_PROMPT =
"Return concise, actionable comments grouped by severity (bug / risk / style). " +
"Do not rewrite the code.";
+// Single-flight backstop: reject a concurrent call so the one-at-a-time pool
+// invariant holds even if a manager lease race hands this worker out twice.
+const POOL_BUSY = "POOL_BUSY";
+let busy = false;
+
// Run `gemini -p <prompt>` with the snippet piped on stdin; resolve its stdout.
function runGemini(code) {
return new Promise((resolve, reject) => {
@@ -41,11 +46,15 @@ server.registerTool(
inputSchema: { code: z.string().describe("The code snippet to review (any language).") },
},
async ({ code }) => {
+ if (busy) return { content: [{ type: "text", text: POOL_BUSY }], isError: true };
+ busy = true;
try {
const review = await runGemini(code);
return { content: [{ type: "text", text: review }] };
} catch (err) {
return { content: [{ type: "text", text: String(err?.message ?? err) }], isError: true };
+ } finally {
+ busy = false;
}
},
);
diff --git a/development/examples/pool-manager-mcp/Dockerfile b/development/examples/pool-manager-mcp/Dockerfile
new file mode 100644
index 00000000..4a615baa
--- /dev/null
+++ b/development/examples/pool-manager-mcp/Dockerfile
@@ -0,0 +1,8 @@
+FROM node:22-slim
+
+WORKDIR /srv
+COPY package.json server.mjs ./
+RUN npm install
+
+EXPOSE 7780
+CMD ["node", "server.mjs"]
diff --git a/development/examples/pool-manager-mcp/package.json b/development/examples/pool-manager-mcp/package.json
new file mode 100644
index 00000000..6888aaf2
--- /dev/null
+++ b/development/examples/pool-manager-mcp/package.json
@@ -0,0 +1,11 @@
+{
+ "name": "pool-manager-mcp",
+ "version": "1.0.0",
+ "private": true,
+ "type": "module",
+ "dependencies": {
+ "@modelcontextprotocol/sdk": "^1.0.0",
+ "express": "^4.19.0",
+ "zod": "^3.23.0"
+ }
+}
diff --git a/development/examples/pool-manager-mcp/sam-node-config.yaml b/development/examples/pool-manager-mcp/sam-node-config.yaml
new file mode 100644
index 00000000..18991131
--- /dev/null
+++ b/development/examples/pool-manager-mcp/sam-node-config.yaml
@@ -0,0 +1,8 @@
+version: "v1alpha1"
+attenuation:
+ policies: []
+services:
+ - type: "mcp"
+ name: "pool-manager"
+ description: "Leases free workers from the code-reviewer pool (acquire/release)"
+ target_url: "http://127.0.0.1:7780/mcp"
diff --git a/development/examples/pool-manager-mcp/server.mjs b/development/examples/pool-manager-mcp/server.mjs
new file mode 100644
index 00000000..c8184660
--- /dev/null
+++ b/development/examples/pool-manager-mcp/server.mjs
@@ -0,0 +1,142 @@
+import express from "express";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import { z } from "zod";
+
+const PORT = Number(process.env.PORT ?? 7780);
+const NODE_URL = process.env.SAM_NODE_URL ?? "http://127.0.0.1:8080/mcp";
+const API_TOKEN = process.env.SAM_API_TOKEN ?? "devtoken";
+const POOL_SERVICE = process.env.SAM_POOL_SERVICE ?? "code-reviewer";
+const DISCOVERY_MS = Number(process.env.SAM_DISCOVERY_MS ?? 3000);
+const LEASE_MS = Number(process.env.SAM_LEASE_MS ?? 60000);
+const GRACE_MISSES = Number(process.env.SAM_GRACE_MISSES ?? 2);
+const ACQUIRE_POLL_MS = 200;
+
+// roster: peer_id -> { tool, leasedUntil, leaseId, missCount }.
+// leasedUntil in the future = busy; leaseId fences stale releases.
+const roster = new Map();
+const now = () => Date.now();
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+let leaseSeq = 0;
+
+// Northbound MCP client to the local node for DHT discovery.
+const node = new Client({ name: "pool-manager", version: "1.0.0" });
+
+// Refresh the roster from DHT discovery. Preserve lease state for known peers;
+// never evict a leased (busy) worker on a transient miss, and only drop a free
+// worker after it has been missing for GRACE_MISSES consecutive passes.
+async function discover() {
+ try {
+ const res = await node.callTool({ name: "find_remote_tools", arguments: {} });
+ const rows = JSON.parse(res.content[0].text); // [{peer_id, tool_name, ...}]
+ const prefix = `mcp://${POOL_SERVICE}/`; // tool names are full URIs, e.g. mcp://code-reviewer/review_code
+ const seen = new Set();
+ for (const r of rows) {
+ if (!r.peer_id || !r.tool_name || !r.tool_name.startsWith(prefix)) continue;
+ seen.add(r.peer_id);
+ const entry = roster.get(r.peer_id) ?? { leasedUntil: 0, leaseId: null, missCount: 0 };
+ entry.tool = r.tool_name;
+ entry.missCount = 0;
+ roster.set(r.peer_id, entry);
+ }
+ for (const [peer, entry] of roster) {
+ if (seen.has(peer)) continue;
+ if (entry.leasedUntil > now()) continue; // busy → keep despite the miss
+ entry.missCount++;
+ if (entry.missCount >= GRACE_MISSES) roster.delete(peer);
+ }
+ } catch (err) {
+ console.error(`discovery failed: ${err?.message ?? err}`);
+ }
+}
+
+// Pick and lease a free worker, or null. Synchronous → collision-free between awaits.
+function leaseFree() {
+ for (const [peer, entry] of roster) {
+ if (entry.leasedUntil <= now()) {
+ entry.leasedUntil = now() + LEASE_MS;
+ entry.leaseId = String(++leaseSeq);
+ return { peer_id: peer, tool: entry.tool, lease_id: entry.leaseId };
+ }
+ }
+ return null;
+}
+
+const server = new McpServer({ name: "pool-manager", version: "1.0.0" });
+
+server.registerTool(
+ "acquire_worker",
+ {
+ description:
+ `Acquire (lease) a free worker from the '${POOL_SERVICE}' pool. Returns {peer_id, tool, lease_id}: ` +
+ "call the tool via call_remote_tool, then release_worker with the same peer_id and lease_id. " +
+ "Blocks up to timeout_secs if all workers are busy; returns {available:false} if none free by then.",
+ inputSchema: { timeout_secs: z.number().optional().describe("Max seconds to wait for a free worker (default 10).") },
+ },
+ async ({ timeout_secs }) => {
+ const deadline = now() + (timeout_secs ?? 10) * 1000;
+ for (;;) {
+ const w = leaseFree();
+ if (w) return { content: [{ type: "text", text: JSON.stringify(w) }] };
+ if (now() >= deadline) return { content: [{ type: "text", text: JSON.stringify({ available: false }) }] };
+ await sleep(ACQUIRE_POLL_MS);
+ }
+ },
+);
+
+server.registerTool(
+ "release_worker",
+ {
+ description:
+ "Release a worker previously acquired with acquire_worker. Pass the lease_id from acquire so a " +
+ "stale release cannot clear a newer lease; released:false means the lease_id did not match.",
+ inputSchema: {
+ peer_id: z.string().describe("Peer id returned by acquire_worker."),
+ lease_id: z.string().optional().describe("Lease id returned by acquire_worker (recommended)."),
+ },
+ },
+ async ({ peer_id, lease_id }) => {
+ const entry = roster.get(peer_id);
+ let released = false;
+ // Fencing: only clear if the caller holds the current lease (or no id given).
+ if (entry && (lease_id === undefined || entry.leaseId === lease_id)) {
+ entry.leasedUntil = 0;
+ entry.leaseId = null;
+ released = true;
+ }
+ return { content: [{ type: "text", text: JSON.stringify({ released }) }] };
+ },
+);
+
+server.registerTool(
+ "list_workers",
+ {
+ description: `List the '${POOL_SERVICE}' pool with per-worker free/busy state.`,
+ inputSchema: {},
+ },
+ async () => {
+ const workers = [...roster.entries()].map(([peer, e]) => ({ peer_id: peer, tool: e.tool, free: e.leasedUntil <= now() }));
+ return { content: [{ type: "text", text: JSON.stringify(workers) }] };
+ },
+);
+
+const app = express();
+app.use(express.json());
+app.post("/mcp", async (req, res) => {
+ const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
+ res.on("close", () => { transport.close(); });
+ await server.connect(transport);
+ await transport.handleRequest(req, res, req.body);
+});
+
+app.listen(PORT, "0.0.0.0", async () => {
+ console.log(`pool-manager MCP server on :${PORT}/mcp (pooling '${POOL_SERVICE}')`);
+ const transport = new StreamableHTTPClientTransport(new URL(NODE_URL), {
+ requestInit: { headers: { Authorization: `Bearer ${API_TOKEN}` } },
+ });
+ await node.connect(transport);
+ await discover();
+ setInterval(discover, DISCOVERY_MS);
+});
diff --git a/development/examples/warm-pool/README.md b/development/examples/warm-pool/README.md
new file mode 100644
index 00000000..b8e8b1bc
--- /dev/null
+++ b/development/examples/warm-pool/README.md
@@ -0,0 +1,124 @@
+# Warm Agent Pool — demo
+
+Fan a batch of files out to a **pool of identical reviewer workers**. A
+**pool-manager** mesh service tracks which workers are free and leases them out;
+an orchestrator (Claude Code, or the headless script here) acquires a worker,
+reviews a file, and releases it. Reviews run in parallel across the pool.
+
+Design: `docs/superpowers/specs/2026-06-28-warm-agent-pool-design.md`.
+
+## Architecture (no gossip, no node changes — just services)
+
+- **Workers:** plain `code-reviewer` mcp services (node-b/c/d). Unmodified.
+- **pool-manager** (node-e): a normal mcp service exposing `acquire_worker` /
+ `release_worker` / `list_workers`. It is also a northbound MCP client of its
+ own node: every few seconds it calls `find_remote_tools(service_name="code-reviewer")`
+ to learn the worker peers (DHT), and it tracks free/busy with **leases**.
+- **Orchestrator:** `acquire_worker` → `call_remote_tool(peer, review_code)` →
+ `release_worker`. Leasing hands each concurrent acquire a different worker, so
+ parallel dispatch never collides.
+
+Why no gossip: the manager gets *who exists* from DHT discovery and *who's busy*
+from its own leases — the two things a readiness broadcast would have provided.
+The tradeoff: leasing is authoritative only while the manager is the sole
+dispatcher (fine for a single-manager pool).
+
+### Concurrency hardening
+
+- **No double-lease:** the manager is single-threaded and `leaseFree()` is
+ synchronous, so two concurrent `acquire_worker` calls can never get the same
+ worker.
+- **Grace eviction:** discovery never drops a *leased* worker on a transient
+ miss, and only drops a free one after `SAM_GRACE_MISSES` consecutive misses —
+ so a slow (busy) worker isn't freed early.
+- **Fencing tokens:** `acquire_worker` returns a `lease_id`; `release_worker`
+ clears the lease only if the id still matches, so a late release from an
+ expired lease can't free a newer holder's worker.
+- **Worker single-flight backstop:** `review_code` returns `POOL_BUSY` if already
+ running; the orchestrator retries. This keeps the one-at-a-time invariant even
+ if a lease race ever slips through.
+
+## Run it
+
+1. **Set a Gemini key** for the reviewer image (free AI Studio key is fine):
+ ```
+ git update-index --skip-worktree development/examples/code-reviewer-mcp/Dockerfile
+ # edit ENV GEMINI_API_KEY=<API_KEY> in that Dockerfile
+ ```
+
+2. Mesh layout ships in `development/kind/mesh-config.yaml`:
+ ```yaml
+ node-a: # orchestrator entry (bare)
+ node-b: code-reviewer-mcp # worker
+ node-c: code-reviewer-mcp # worker
+ node-d: code-reviewer-mcp # worker
+ node-e: pool-manager-mcp # manager
+ ```
+
+3. Bring the mesh up, then port-forward the caller node:
+ ```
+ make kind
+ kubectl --context kind-sam-kind -n sam-kind port-forward deploy/node-a 9091:8080
+ ```
+
+### Option A — drive it from Claude Code (the cool one)
+
+Register node-a as an MCP server (`.mcp.json` in the project root):
+```json
+{
+ "mcpServers": {
+ "sam-mesh": {
+ "type": "http",
+ "url": "http://127.0.0.1:9091/mcp",
+ "headers": { "Authorization": "Bearer devtoken" }
+ }
+ }
+}
+```
+Reconnect Claude Code, then prompt:
+> Using the sam-mesh tools: find the pool-manager, then for each file in
+> `development/examples/warm-pool/samples/` acquire a worker, call its
+> `review_code` with the file contents, and release it (pass the lease_id back
+> to release_worker). Run them in parallel.
+
+Claude fires parallel `acquire_worker` → `call_remote_tool` → `release_worker`
+chains; you watch the reviews come back concurrently.
+
+### Option B — headless script
+
+```
+cd development/examples/warm-pool
+npm install
+node orchestrator.mjs samples
+```
+Example output:
+```
+queued 3 review task(s)
+pool-manager at 9c2ab1f0
+[+0.4s] -> 3f9a1c2e samples/auth.js
+[+0.4s] -> b7d4e0a1 samples/parse.py
+[+0.4s] -> c1a8f552 samples/rates.go
+[+6.1s] <- b7d4e0a1 samples/parse.py (712 chars)
+[+7.0s] <- 3f9a1c2e samples/auth.js (901 chars)
+```
+
+## Elasticity beat (add a worker mid-job)
+
+Scale a reviewer down, start a larger job, then scale it back up — the manager
+picks the new worker up on its next discovery pass and starts leasing it:
+```
+kubectl --context kind-sam-kind -n sam-kind scale deploy/node-d --replicas=0
+# start a big job, then:
+kubectl --context kind-sam-kind -n sam-kind scale deploy/node-d --replicas=1
+```
+
+## Config (env vars)
+
+| var | default | used by |
+|-----|---------|---------|
+| `SAM_NODE_URL` | manager: `http://127.0.0.1:8080/mcp`, orchestrator: `http://127.0.0.1:9091/mcp` | both |
+| `SAM_API_TOKEN` | `devtoken` | both |
+| `SAM_POOL_SERVICE` | `code-reviewer` | manager |
+| `SAM_DISCOVERY_MS` | `3000` | manager |
+| `SAM_LEASE_MS` | `60000` | manager |
+| `SAM_ACQUIRE_TIMEOUT` | `60` | orchestrator |
diff --git a/development/examples/warm-pool/orchestrator.mjs b/development/examples/warm-pool/orchestrator.mjs
new file mode 100644
index 00000000..2bba792f
--- /dev/null
+++ b/development/examples/warm-pool/orchestrator.mjs
@@ -0,0 +1,87 @@
+// Headless warm-pool orchestrator (alternative to driving it from Claude Code).
+// For each file: acquire a free worker from the pool-manager, review, release.
+// All tasks run concurrently; the manager's blocking acquire + leasing serialize
+// dispatch to the pool size, so no local queue is needed.
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import { readFileSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+const NODE_URL = process.env.SAM_NODE_URL ?? "http://127.0.0.1:9091/mcp";
+const API_TOKEN = process.env.SAM_API_TOKEN ?? "devtoken";
+const ACQUIRE_TIMEOUT = Number(process.env.SAM_ACQUIRE_TIMEOUT ?? 60);
+
+function collectFiles(paths) {
+ const out = [];
+ for (const p of paths) {
+ if (statSync(p).isDirectory()) {
+ for (const f of readdirSync(p)) {
+ const fp = join(p, f);
+ if (statSync(fp).isFile()) out.push(fp);
+ }
+ } else {
+ out.push(p);
+ }
+ }
+ return out;
+}
+
+const short = (peer) => peer.slice(-8);
+
+async function main() {
+ const files = collectFiles(process.argv.slice(2));
+ if (!files.length) {
+ console.error("usage: node orchestrator.mjs <file|dir> ...");
+ process.exit(1);
+ }
+ const tasks = files.map((f) => ({ name: f, code: readFileSync(f, "utf8") }));
+ console.log(`queued ${tasks.length} review task(s)`);
+
+ const client = new Client({ name: "warm-pool-orchestrator", version: "1.0.0" });
+ const transport = new StreamableHTTPClientTransport(new URL(NODE_URL), {
+ requestInit: { headers: { Authorization: `Bearer ${API_TOKEN}` } },
+ });
+ await client.connect(transport);
+
+ const call = async (peer, tool, args) => {
+ const res = await client.callTool({ name: "call_remote_tool", arguments: { peer_id: peer, tool_name: tool, arguments: args } });
+ return res?.content?.[0]?.text ?? "";
+ };
+
+ // Locate the pool-manager peer (tool names are full URIs).
+ const found = JSON.parse((await client.callTool({ name: "find_remote_tools", arguments: {} })).content[0].text);
+ const manager = found.find((r) => r.tool_name === "mcp://pool-manager/acquire_worker")?.peer_id;
+ if (!manager) { console.error("no pool-manager found in the mesh"); process.exit(1); }
+ console.log(`pool-manager at ${short(manager)}`);
+
+ const t0 = Date.now();
+ const elapsed = () => ((Date.now() - t0) / 1000).toFixed(1);
+
+ const process1 = async (task) => {
+ for (;;) {
+ const acq = JSON.parse(await call(manager, "mcp://pool-manager/acquire_worker", { timeout_secs: ACQUIRE_TIMEOUT }));
+ if (!acq.peer_id) { console.error(`[+${elapsed()}s] no worker for ${task.name}`); return null; }
+ console.log(`[+${elapsed()}s] -> ${short(acq.peer_id)} ${task.name}`);
+ let review;
+ try {
+ review = await call(acq.peer_id, acq.tool, { code: task.code });
+ } finally {
+ await call(manager, "mcp://pool-manager/release_worker", { peer_id: acq.peer_id, lease_id: acq.lease_id });
+ }
+ // Worker single-flight backstop tripped; the lease race is rare, so retry.
+ if (review.trim() === "POOL_BUSY") { console.log(`[+${elapsed()}s] ~~ ${short(acq.peer_id)} busy, retrying ${task.name}`); continue; }
+ console.log(`[+${elapsed()}s] <- ${short(acq.peer_id)} ${task.name} (${review.length} chars)`);
+ return { ...task, peer: acq.peer_id, review };
+ }
+ };
+
+ const results = (await Promise.all(tasks.map(process1))).filter(Boolean);
+
+ console.log(`\n=== done in ${elapsed()}s: ${results.length} review(s) across ${new Set(results.map((r) => r.peer)).size} worker(s) ===`);
+ for (const r of results) {
+ console.log(`\n----- ${r.name} (reviewed by ${short(r.peer)}) -----\n${r.review}`);
+ }
+ await client.close();
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });
diff --git a/development/examples/warm-pool/package.json b/development/examples/warm-pool/package.json
new file mode 100644
index 00000000..05df8848
--- /dev/null
+++ b/development/examples/warm-pool/package.json
@@ -0,0 +1,9 @@
+{
+ "name": "warm-pool-orchestrator",
+ "version": "1.0.0",
+ "private": true,
+ "type": "module",
+ "dependencies": {
+ "@modelcontextprotocol/sdk": "^1.0.0"
+ }
+}
diff --git a/development/examples/warm-pool/samples/auth.js b/development/examples/warm-pool/samples/auth.js
new file mode 100644
index 00000000..a8debf97
--- /dev/null
+++ b/development/examples/warm-pool/samples/auth.js
@@ -0,0 +1,16 @@
+// Sample file with intentional issues for the reviewer pool to find.
+const sessions = {};
+
+function login(user, password) {
+ if (password == user.password) {
+ const token = Math.random().toString(36);
+ sessions[token] = user.name;
+ return token;
+ }
+}
+
+function whoami(token) {
+ return sessions[token];
+}
+
+module.exports = { login, whoami };
diff --git a/development/examples/warm-pool/samples/parse.py b/development/examples/warm-pool/samples/parse.py
new file mode 100644
index 00000000..7d1a8355
--- /dev/null
+++ b/development/examples/warm-pool/samples/parse.py
@@ -0,0 +1,10 @@
+# Sample file with intentional issues for the reviewer pool to find.
+def parse_range(spec):
+ start, end = spec.split("-")
+ return list(range(int(start), int(end)))
+
+
+def load(path):
+ f = open(path)
+ data = f.read()
+ return [parse_range(line) for line in data.split(",")]
diff --git a/development/examples/warm-pool/samples/rates.go b/development/examples/warm-pool/samples/rates.go
new file mode 100644
index 00000000..94401627
--- /dev/null
+++ b/development/examples/warm-pool/samples/rates.go
@@ -0,0 +1,17 @@
+// Sample file with intentional issues for the reviewer pool to find.
+package rates
+
+import "sync"
+
+var cache = map[string]float64{}
+var mu sync.Mutex
+
+func Set(k string, v float64) {
+ mu.Lock()
+ cache[k] = v
+ mu.Unlock()
+}
+
+func Get(k string) float64 {
+ return cache[k]
+}
diff --git a/development/kind/10-control-plane.yaml b/development/kind/10-control-plane.yaml
index 5938a8ef..ed98bfa4 100644
--- a/development/kind/10-control-plane.yaml
+++ b/development/kind/10-control-plane.yaml
@@ -138,6 +138,10 @@ data:
role: "sam-admin"
- members: ["user:system:serviceaccount:${NAMESPACE}:node-c-sa"]
role: "sam-admin"
+ - members: ["user:system:serviceaccount:${NAMESPACE}:node-d-sa"]
+ role: "sam-admin"
+ - members: ["user:system:serviceaccount:${NAMESPACE}:node-e-sa"]
+ role: "sam-admin"
- members: ["user:system:serviceaccount:${NAMESPACE}:local-node-sa"]
role: "sam-admin"
- members: ["user:system:serviceaccount:${NAMESPACE}:sam-box-sa"]
diff --git a/development/kind/kind-config.yaml b/development/kind/kind-config.yaml
index e679fa67..8296eea6 100644
--- a/development/kind/kind-config.yaml
+++ b/development/kind/kind-config.yaml
@@ -29,3 +29,9 @@ nodes:
- role: worker
labels:
sam-role: node-c
+ - role: worker
+ labels:
+ sam-role: node-d
+ - role: worker
+ labels:
+ sam-role: node-e
diff --git a/development/kind/mesh-config.yaml b/development/kind/mesh-config.yaml
index 138bc260..78ac5ab5 100644
--- a/development/kind/mesh-config.yaml
+++ b/development/kind/mesh-config.yaml
@@ -1,5 +1,9 @@
# node -> service. A blank value means a bare node (no service, e.g. a caller).
# The service value is a folder name under development/examples/.
+# node-a is the orchestrator entry; node-b/c/d form a warm reviewer pool;
+# node-e runs the pool-manager that leases those workers out.
node-a:
-node-b: calc-mcp
-node-c: greeter-mcp
+node-b: code-reviewer-mcp
+node-c: code-reviewer-mcp
+node-d: code-reviewer-mcp
+node-e: pool-manager-mcp