Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
development/examples: add warm agent pool (pool-manager + reviewer pool)
Tomas Tormoef30fe0f8915
diff --git a/development/examples/code-reviewer-mcp/review_server.mjs b/development/examples/code-reviewer-mcp/review_server.mjsindex 3ea44f05..ef0486b0 100644--- a/development/examples/code-reviewer-mcp/review_server.mjs+++ b/development/examples/code-reviewer-mcp/review_server.mjs@@ -10,6 +10,11 @@ const REVIEW_PROMPT ="Return concise, actionable comments grouped by severity (bug / risk / style). " +"Do not rewrite the code.";+// Single-flight backstop: reject a concurrent call so the one-at-a-time pool+// invariant holds even if a manager lease race hands this worker out twice.+const POOL_BUSY = "POOL_BUSY";+let busy = false;+// Run `gemini -p <prompt>` with the snippet piped on stdin; resolve its stdout.function runGemini(code) {return new Promise((resolve, reject) => {@@ -41,11 +46,15 @@ server.registerTool(inputSchema: { code: z.string().describe("The code snippet to review (any language).") },},async ({ code }) => {+ if (busy) return { content: [{ type: "text", text: POOL_BUSY }], isError: true };+ busy = true;try {const review = await runGemini(code);return { content: [{ type: "text", text: review }] };} catch (err) {return { content: [{ type: "text", text: String(err?.message ?? err) }], isError: true };+ } finally {+ busy = false;}},);diff --git a/development/examples/pool-manager-mcp/Dockerfile b/development/examples/pool-manager-mcp/Dockerfilenew file mode 100644index 00000000..4a615baa--- /dev/null+++ b/development/examples/pool-manager-mcp/Dockerfile@@ -0,0 +1,8 @@+FROM node:22-slim++WORKDIR /srv+COPY package.json server.mjs ./+RUN npm install++EXPOSE 7780+CMD ["node", "server.mjs"]diff --git a/development/examples/pool-manager-mcp/package.json b/development/examples/pool-manager-mcp/package.jsonnew file mode 100644index 00000000..6888aaf2--- /dev/null+++ b/development/examples/pool-manager-mcp/package.json@@ -0,0 +1,11 @@+{+ "name": "pool-manager-mcp",+ "version": "1.0.0",+ "private": true,+ "type": "module",+ "dependencies": {+ "@modelcontextprotocol/sdk": "^1.0.0",+ "express": "^4.19.0",+ "zod": "^3.23.0"+ }+}diff --git a/development/examples/pool-manager-mcp/sam-node-config.yaml b/development/examples/pool-manager-mcp/sam-node-config.yamlnew file mode 100644index 00000000..18991131--- /dev/null+++ b/development/examples/pool-manager-mcp/sam-node-config.yaml@@ -0,0 +1,8 @@+version: "v1alpha1"+attenuation:+ policies: []+services:+ - type: "mcp"+ name: "pool-manager"+ description: "Leases free workers from the code-reviewer pool (acquire/release)"+ target_url: "http://127.0.0.1:7780/mcp"diff --git a/development/examples/pool-manager-mcp/server.mjs b/development/examples/pool-manager-mcp/server.mjsnew file mode 100644index 00000000..c8184660--- /dev/null+++ b/development/examples/pool-manager-mcp/server.mjs@@ -0,0 +1,142 @@+import express from "express";+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";+import { Client } from "@modelcontextprotocol/sdk/client/index.js";+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";+import { z } from "zod";++const PORT = Number(process.env.PORT ?? 7780);+const NODE_URL = process.env.SAM_NODE_URL ?? "http://127.0.0.1:8080/mcp";+const API_TOKEN = process.env.SAM_API_TOKEN ?? "devtoken";+const POOL_SERVICE = process.env.SAM_POOL_SERVICE ?? "code-reviewer";+const DISCOVERY_MS = Number(process.env.SAM_DISCOVERY_MS ?? 3000);+const LEASE_MS = Number(process.env.SAM_LEASE_MS ?? 60000);+const GRACE_MISSES = Number(process.env.SAM_GRACE_MISSES ?? 2);+const ACQUIRE_POLL_MS = 200;++// roster: peer_id -> { tool, leasedUntil, leaseId, missCount }.+// leasedUntil in the future = busy; leaseId fences stale releases.+const roster = new Map();+const now = () => Date.now();+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));+let leaseSeq = 0;++// Northbound MCP client to the local node for DHT discovery.+const node = new Client({ name: "pool-manager", version: "1.0.0" });++// Refresh the roster from DHT discovery. Preserve lease state for known peers;+// never evict a leased (busy) worker on a transient miss, and only drop a free+// worker after it has been missing for GRACE_MISSES consecutive passes.+async function discover() {+ try {+ const res = await node.callTool({ name: "find_remote_tools", arguments: {} });+ const rows = JSON.parse(res.content[0].text); // [{peer_id, tool_name, ...}]+ const prefix = `mcp://${POOL_SERVICE}/`; // tool names are full URIs, e.g. mcp://code-reviewer/review_code+ const seen = new Set();+ for (const r of rows) {+ if (!r.peer_id || !r.tool_name || !r.tool_name.startsWith(prefix)) continue;+ seen.add(r.peer_id);+ const entry = roster.get(r.peer_id) ?? { leasedUntil: 0, leaseId: null, missCount: 0 };+ entry.tool = r.tool_name;+ entry.missCount = 0;+ roster.set(r.peer_id, entry);+ }+ for (const [peer, entry] of roster) {+ if (seen.has(peer)) continue;+ if (entry.leasedUntil > now()) continue; // busy → keep despite the miss+ entry.missCount++;+ if (entry.missCount >= GRACE_MISSES) roster.delete(peer);+ }+ } catch (err) {+ console.error(`discovery failed: ${err?.message ?? err}`);+ }+}++// Pick and lease a free worker, or null. Synchronous → collision-free between awaits.+function leaseFree() {+ for (const [peer, entry] of roster) {+ if (entry.leasedUntil <= now()) {+ entry.leasedUntil = now() + LEASE_MS;+ entry.leaseId = String(++leaseSeq);+ return { peer_id: peer, tool: entry.tool, lease_id: entry.leaseId };+ }+ }+ return null;+}++const server = new McpServer({ name: "pool-manager", version: "1.0.0" });++server.registerTool(+ "acquire_worker",+ {+ description:+ `Acquire (lease) a free worker from the '${POOL_SERVICE}' pool. Returns {peer_id, tool, lease_id}: ` ++ "call the tool via call_remote_tool, then release_worker with the same peer_id and lease_id. " ++ "Blocks up to timeout_secs if all workers are busy; returns {available:false} if none free by then.",+ inputSchema: { timeout_secs: z.number().optional().describe("Max seconds to wait for a free worker (default 10).") },+ },+ async ({ timeout_secs }) => {+ const deadline = now() + (timeout_secs ?? 10) * 1000;+ for (;;) {+ const w = leaseFree();+ if (w) return { content: [{ type: "text", text: JSON.stringify(w) }] };+ if (now() >= deadline) return { content: [{ type: "text", text: JSON.stringify({ available: false }) }] };+ await sleep(ACQUIRE_POLL_MS);+ }+ },+);++server.registerTool(+ "release_worker",+ {+ description:+ "Release a worker previously acquired with acquire_worker. Pass the lease_id from acquire so a " ++ "stale release cannot clear a newer lease; released:false means the lease_id did not match.",+ inputSchema: {+ peer_id: z.string().describe("Peer id returned by acquire_worker."),+ lease_id: z.string().optional().describe("Lease id returned by acquire_worker (recommended)."),+ },+ },+ async ({ peer_id, lease_id }) => {+ const entry = roster.get(peer_id);+ let released = false;+ // Fencing: only clear if the caller holds the current lease (or no id given).+ if (entry && (lease_id === undefined || entry.leaseId === lease_id)) {+ entry.leasedUntil = 0;+ entry.leaseId = null;+ released = true;+ }+ return { content: [{ type: "text", text: JSON.stringify({ released }) }] };+ },+);++server.registerTool(+ "list_workers",+ {+ description: `List the '${POOL_SERVICE}' pool with per-worker free/busy state.`,+ inputSchema: {},+ },+ async () => {+ const workers = [...roster.entries()].map(([peer, e]) => ({ peer_id: peer, tool: e.tool, free: e.leasedUntil <= now() }));+ return { content: [{ type: "text", text: JSON.stringify(workers) }] };+ },+);++const app = express();+app.use(express.json());+app.post("/mcp", async (req, res) => {+ const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });+ res.on("close", () => { transport.close(); });+ await server.connect(transport);+ await transport.handleRequest(req, res, req.body);+});++app.listen(PORT, "0.0.0.0", async () => {+ console.log(`pool-manager MCP server on :${PORT}/mcp (pooling '${POOL_SERVICE}')`);+ const transport = new StreamableHTTPClientTransport(new URL(NODE_URL), {+ requestInit: { headers: { Authorization: `Bearer ${API_TOKEN}` } },+ });+ await node.connect(transport);+ await discover();+ setInterval(discover, DISCOVERY_MS);+});diff --git a/development/examples/warm-pool/README.md b/development/examples/warm-pool/README.mdnew file mode 100644index 00000000..b8e8b1bc--- /dev/null+++ b/development/examples/warm-pool/README.md@@ -0,0 +1,124 @@+# Warm Agent Pool — demo++Fan a batch of files out to a **pool of identical reviewer workers**. A+**pool-manager** mesh service tracks which workers are free and leases them out;+an orchestrator (Claude Code, or the headless script here) acquires a worker,+reviews a file, and releases it. Reviews run in parallel across the pool.++Design: `docs/superpowers/specs/2026-06-28-warm-agent-pool-design.md`.++## Architecture (no gossip, no node changes — just services)++- **Workers:** plain `code-reviewer` mcp services (node-b/c/d). Unmodified.+- **pool-manager** (node-e): a normal mcp service exposing `acquire_worker` /+ `release_worker` / `list_workers`. It is also a northbound MCP client of its+ own node: every few seconds it calls `find_remote_tools(service_name="code-reviewer")`+ to learn the worker peers (DHT), and it tracks free/busy with **leases**.+- **Orchestrator:** `acquire_worker` → `call_remote_tool(peer, review_code)` →+ `release_worker`. Leasing hands each concurrent acquire a different worker, so+ parallel dispatch never collides.++Why no gossip: the manager gets *who exists* from DHT discovery and *who's busy*+from its own leases — the two things a readiness broadcast would have provided.+The tradeoff: leasing is authoritative only while the manager is the sole+dispatcher (fine for a single-manager pool).++### Concurrency hardening++- **No double-lease:** the manager is single-threaded and `leaseFree()` is+ synchronous, so two concurrent `acquire_worker` calls can never get the same+ worker.+- **Grace eviction:** discovery never drops a *leased* worker on a transient+ miss, and only drops a free one after `SAM_GRACE_MISSES` consecutive misses —+ so a slow (busy) worker isn't freed early.+- **Fencing tokens:** `acquire_worker` returns a `lease_id`; `release_worker`+ clears the lease only if the id still matches, so a late release from an+ expired lease can't free a newer holder's worker.+- **Worker single-flight backstop:** `review_code` returns `POOL_BUSY` if already+ running; the orchestrator retries. This keeps the one-at-a-time invariant even+ if a lease race ever slips through.++## Run it++1. **Set a Gemini key** for the reviewer image (free AI Studio key is fine):+ ```+ git update-index --skip-worktree development/examples/code-reviewer-mcp/Dockerfile+ # edit ENV GEMINI_API_KEY=<API_KEY> in that Dockerfile+ ```++2. Mesh layout ships in `development/kind/mesh-config.yaml`:+ ```yaml+ node-a: # orchestrator entry (bare)+ node-b: code-reviewer-mcp # worker+ node-c: code-reviewer-mcp # worker+ node-d: code-reviewer-mcp # worker+ node-e: pool-manager-mcp # manager+ ```++3. Bring the mesh up, then port-forward the caller node:+ ```+ make kind+ kubectl --context kind-sam-kind -n sam-kind port-forward deploy/node-a 9091:8080+ ```++### Option A — drive it from Claude Code (the cool one)++Register node-a as an MCP server (`.mcp.json` in the project root):+```json+{+ "mcpServers": {+ "sam-mesh": {+ "type": "http",+ "url": "http://127.0.0.1:9091/mcp",+ "headers": { "Authorization": "Bearer devtoken" }+ }+ }+}+```+Reconnect Claude Code, then prompt:+> Using the sam-mesh tools: find the pool-manager, then for each file in+> `development/examples/warm-pool/samples/` acquire a worker, call its+> `review_code` with the file contents, and release it (pass the lease_id back+> to release_worker). Run them in parallel.++Claude fires parallel `acquire_worker` → `call_remote_tool` → `release_worker`+chains; you watch the reviews come back concurrently.++### Option B — headless script++```+cd development/examples/warm-pool+npm install+node orchestrator.mjs samples+```+Example output:+```+queued 3 review task(s)+pool-manager at 9c2ab1f0+[+0.4s] -> 3f9a1c2e samples/auth.js+[+0.4s] -> b7d4e0a1 samples/parse.py+[+0.4s] -> c1a8f552 samples/rates.go+[+6.1s] <- b7d4e0a1 samples/parse.py (712 chars)+[+7.0s] <- 3f9a1c2e samples/auth.js (901 chars)+```++## Elasticity beat (add a worker mid-job)++Scale a reviewer down, start a larger job, then scale it back up — the manager+picks the new worker up on its next discovery pass and starts leasing it:+```+kubectl --context kind-sam-kind -n sam-kind scale deploy/node-d --replicas=0+# start a big job, then:+kubectl --context kind-sam-kind -n sam-kind scale deploy/node-d --replicas=1+```++## Config (env vars)++| var | default | used by |+|-----|---------|---------|+| `SAM_NODE_URL` | manager: `http://127.0.0.1:8080/mcp`, orchestrator: `http://127.0.0.1:9091/mcp` | both |+| `SAM_API_TOKEN` | `devtoken` | both |+| `SAM_POOL_SERVICE` | `code-reviewer` | manager |+| `SAM_DISCOVERY_MS` | `3000` | manager |+| `SAM_LEASE_MS` | `60000` | manager |+| `SAM_ACQUIRE_TIMEOUT` | `60` | orchestrator |diff --git a/development/examples/warm-pool/orchestrator.mjs b/development/examples/warm-pool/orchestrator.mjsnew file mode 100644index 00000000..2bba792f--- /dev/null+++ b/development/examples/warm-pool/orchestrator.mjs@@ -0,0 +1,87 @@+// Headless warm-pool orchestrator (alternative to driving it from Claude Code).+// For each file: acquire a free worker from the pool-manager, review, release.+// All tasks run concurrently; the manager's blocking acquire + leasing serialize+// dispatch to the pool size, so no local queue is needed.+import { Client } from "@modelcontextprotocol/sdk/client/index.js";+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";+import { readFileSync, readdirSync, statSync } from "node:fs";+import { join } from "node:path";++const NODE_URL = process.env.SAM_NODE_URL ?? "http://127.0.0.1:9091/mcp";+const API_TOKEN = process.env.SAM_API_TOKEN ?? "devtoken";+const ACQUIRE_TIMEOUT = Number(process.env.SAM_ACQUIRE_TIMEOUT ?? 60);++function collectFiles(paths) {+ const out = [];+ for (const p of paths) {+ if (statSync(p).isDirectory()) {+ for (const f of readdirSync(p)) {+ const fp = join(p, f);+ if (statSync(fp).isFile()) out.push(fp);+ }+ } else {+ out.push(p);+ }+ }+ return out;+}++const short = (peer) => peer.slice(-8);++async function main() {+ const files = collectFiles(process.argv.slice(2));+ if (!files.length) {+ console.error("usage: node orchestrator.mjs <file|dir> ...");+ process.exit(1);+ }+ const tasks = files.map((f) => ({ name: f, code: readFileSync(f, "utf8") }));+ console.log(`queued ${tasks.length} review task(s)`);++ const client = new Client({ name: "warm-pool-orchestrator", version: "1.0.0" });+ const transport = new StreamableHTTPClientTransport(new URL(NODE_URL), {+ requestInit: { headers: { Authorization: `Bearer ${API_TOKEN}` } },+ });+ await client.connect(transport);++ const call = async (peer, tool, args) => {+ const res = await client.callTool({ name: "call_remote_tool", arguments: { peer_id: peer, tool_name: tool, arguments: args } });+ return res?.content?.[0]?.text ?? "";+ };++ // Locate the pool-manager peer (tool names are full URIs).+ const found = JSON.parse((await client.callTool({ name: "find_remote_tools", arguments: {} })).content[0].text);+ const manager = found.find((r) => r.tool_name === "mcp://pool-manager/acquire_worker")?.peer_id;+ if (!manager) { console.error("no pool-manager found in the mesh"); process.exit(1); }+ console.log(`pool-manager at ${short(manager)}`);++ const t0 = Date.now();+ const elapsed = () => ((Date.now() - t0) / 1000).toFixed(1);++ const process1 = async (task) => {+ for (;;) {+ const acq = JSON.parse(await call(manager, "mcp://pool-manager/acquire_worker", { timeout_secs: ACQUIRE_TIMEOUT }));+ if (!acq.peer_id) { console.error(`[+${elapsed()}s] no worker for ${task.name}`); return null; }+ console.log(`[+${elapsed()}s] -> ${short(acq.peer_id)} ${task.name}`);+ let review;+ try {+ review = await call(acq.peer_id, acq.tool, { code: task.code });+ } finally {+ await call(manager, "mcp://pool-manager/release_worker", { peer_id: acq.peer_id, lease_id: acq.lease_id });+ }+ // Worker single-flight backstop tripped; the lease race is rare, so retry.+ if (review.trim() === "POOL_BUSY") { console.log(`[+${elapsed()}s] ~~ ${short(acq.peer_id)} busy, retrying ${task.name}`); continue; }+ console.log(`[+${elapsed()}s] <- ${short(acq.peer_id)} ${task.name} (${review.length} chars)`);+ return { ...task, peer: acq.peer_id, review };+ }+ };++ const results = (await Promise.all(tasks.map(process1))).filter(Boolean);++ console.log(`\n=== done in ${elapsed()}s: ${results.length} review(s) across ${new Set(results.map((r) => r.peer)).size} worker(s) ===`);+ for (const r of results) {+ console.log(`\n----- ${r.name} (reviewed by ${short(r.peer)}) -----\n${r.review}`);+ }+ await client.close();+}++main().catch((e) => { console.error(e); process.exit(1); });diff --git a/development/examples/warm-pool/package.json b/development/examples/warm-pool/package.jsonnew file mode 100644index 00000000..05df8848--- /dev/null+++ b/development/examples/warm-pool/package.json@@ -0,0 +1,9 @@+{+ "name": "warm-pool-orchestrator",+ "version": "1.0.0",+ "private": true,+ "type": "module",+ "dependencies": {+ "@modelcontextprotocol/sdk": "^1.0.0"+ }+}diff --git a/development/examples/warm-pool/samples/auth.js b/development/examples/warm-pool/samples/auth.jsnew file mode 100644index 00000000..a8debf97--- /dev/null+++ b/development/examples/warm-pool/samples/auth.js@@ -0,0 +1,16 @@+// Sample file with intentional issues for the reviewer pool to find.+const sessions = {};++function login(user, password) {+ if (password == user.password) {+ const token = Math.random().toString(36);+ sessions[token] = user.name;+ return token;+ }+}++function whoami(token) {+ return sessions[token];+}++module.exports = { login, whoami };diff --git a/development/examples/warm-pool/samples/parse.py b/development/examples/warm-pool/samples/parse.pynew file mode 100644index 00000000..7d1a8355--- /dev/null+++ b/development/examples/warm-pool/samples/parse.py@@ -0,0 +1,10 @@+# Sample file with intentional issues for the reviewer pool to find.+def parse_range(spec):+ start, end = spec.split("-")+ return list(range(int(start), int(end)))+++def load(path):+ f = open(path)+ data = f.read()+ return [parse_range(line) for line in data.split(",")]diff --git a/development/examples/warm-pool/samples/rates.go b/development/examples/warm-pool/samples/rates.gonew file mode 100644index 00000000..94401627--- /dev/null+++ b/development/examples/warm-pool/samples/rates.go@@ -0,0 +1,17 @@+// Sample file with intentional issues for the reviewer pool to find.+package rates++import "sync"++var cache = map[string]float64{}+var mu sync.Mutex++func Set(k string, v float64) {+ mu.Lock()+ cache[k] = v+ mu.Unlock()+}++func Get(k string) float64 {+ return cache[k]+}diff --git a/development/kind/10-control-plane.yaml b/development/kind/10-control-plane.yamlindex 5938a8ef..ed98bfa4 100644--- a/development/kind/10-control-plane.yaml+++ b/development/kind/10-control-plane.yaml@@ -138,6 +138,10 @@ data:role: "sam-admin"- members: ["user:system:serviceaccount:${NAMESPACE}:node-c-sa"]role: "sam-admin"+ - members: ["user:system:serviceaccount:${NAMESPACE}:node-d-sa"]+ role: "sam-admin"+ - members: ["user:system:serviceaccount:${NAMESPACE}:node-e-sa"]+ role: "sam-admin"- members: ["user:system:serviceaccount:${NAMESPACE}:local-node-sa"]role: "sam-admin"- members: ["user:system:serviceaccount:${NAMESPACE}:sam-box-sa"]diff --git a/development/kind/kind-config.yaml b/development/kind/kind-config.yamlindex e679fa67..8296eea6 100644--- a/development/kind/kind-config.yaml+++ b/development/kind/kind-config.yaml@@ -29,3 +29,9 @@ nodes:- role: workerlabels:sam-role: node-c+ - role: worker+ labels:+ sam-role: node-d+ - role: worker+ labels:+ sam-role: node-ediff --git a/development/kind/mesh-config.yaml b/development/kind/mesh-config.yamlindex 138bc260..78ac5ab5 100644--- a/development/kind/mesh-config.yaml+++ b/development/kind/mesh-config.yaml@@ -1,5 +1,9 @@# node -> service. A blank value means a bare node (no service, e.g. a caller).# The service value is a folder name under development/examples/.+# node-a is the orchestrator entry; node-b/c/d form a warm reviewer pool;+# node-e runs the pool-manager that leases those workers out.node-a:-node-b: calc-mcp-node-c: greeter-mcp+node-b: code-reviewer-mcp+node-c: code-reviewer-mcp+node-d: code-reviewer-mcp+node-e: pool-manager-mcp