Skip to content

Agent Mesh

sdk: pull from the control plane before admitting routers

Antonio Ojea01e7302e387a
diff --git a/sdk/README.md b/sdk/README.md
index 0affbdd5..ec438b83 100644
--- a/sdk/README.md
+++ b/sdk/README.md
@@ -50,7 +50,7 @@ Milestones 1 to 5 are implemented and tested in both languages:
| The mesh as a transport for HTTP clients: `session.fetch()` for fetch-based clients, `MeshTransport` for httpx, so the official A2A SDK's client works unchanged | `fetch` | httpx |
| Provider authorizer: the baseline Datalog and the mesh policy (`GET /policies`), evaluated as `sam-node` does | yes | yes |
| A2A ingress for the agent: `/libp2p-http` for `a2a://<name>`, forwarded to an A2A server beside the process or answered in it; not announced anywhere | yes | yes |
-| Control plane pull on `sam-node`'s interval: `/keys` verified against the trusted set, credential refresh after a rotation, `/info` bans and router addresses | yes | yes |
+| Control plane pull before join and on `sam-node`'s interval: `/keys` verified against the trusted set, credential refresh after a rotation, `/info` bans and router addresses | yes | yes |
| Gossip events from the control plane (`/sam/mesh/events/v1`, StrictSign): ban enforced at once, key rotation adopted, policy update pulls | yes | yes |
| Banned peers refused: connections dropped and denied, handshakes and requests refused, dials refused | yes | yes |
| Runs in a browser page: WebSocket and Noise to the router, state in IndexedDB, the agent answered by a fetch handler; `sdk/js/examples/browser` against `sam-one`, tested in Chromium | yes | — |
diff --git a/sdk/js/src/session.test.ts b/sdk/js/src/session.test.ts
index 6b5313ed..dead2fba 100644
--- a/sdk/js/src/session.test.ts
+++ b/sdk/js/src/session.test.ts
@@ -29,11 +29,14 @@ import { tls } from "@libp2p/tls";
import { multiaddr } from "@multiformats/multiaddr";
import { createLibp2p } from "libp2p";
import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
import { after, before, test } from "node:test";
import { AUTH_HANDLER_OPTIONS, AUTH_PROTOCOL, AuthRejectedError, authenticateWithPeer, authStreamHandler } from "./auth.ts";
import { BiscuitVerificationError, ROLE_ROUTER, loadBiscuit, verifyPeerBiscuit } from "./biscuit.ts";
import { ROLE_NODE } from "./controlplane.ts";
-import { BootstrapEnrollRequestSchema, BootstrapEnrollResponseSchema, ControlPlaneInfoResponseSchema, EnrollmentStatus, KeysResponseSchema, AuthFrameSchema } from "./gen/sam_pb.ts";
+import { BootstrapEnrollRequestSchema, BootstrapEnrollResponseSchema, ControlPlaneInfoResponseSchema, EnrollmentStatus, KeysResponseSchema, AuthFrameSchema, TokenRefreshRequestSchema, TokenRefreshResponseSchema } from "./gen/sam_pb.ts";
import { Identity } from "./identity.ts";
import { HTTP_HANDLER_OPTIONS, HTTP_PROTOCOL, a2aEndpoint, httpIngressHandler } from "./libp2p-http.ts";
import { LabelsNotSatisfiedError } from "./mcp.ts";
@@ -41,15 +44,24 @@ import { AgentMesh } from "./mesh.ts";
import { MeshSession, type JoinOptions } from "./session.ts";
type Wasm = Awaited<ReturnType<typeof loadBiscuit>>;
+type KeyPair = InstanceType<Wasm["KeyPair"]>;
let wasm: Wasm;
-let cpKeyPair: InstanceType<Wasm["KeyPair"]>;
+// The control plane's signing key, as an Identity so a test can sign /keys
+// with it and as the biscuit key pair that mints credentials.
+const cpIdentity = Identity.generate();
+let cpKeyPair: KeyPair;
let cpKey: Uint8Array;
let router: Libp2p;
let routerAddr: string;
let routerBiscuit: Uint8Array;
-function mint(peerId: string, role: string, expiration = "2035-01-01T00:00:00Z", labels: Record<string, string> = {}): Uint8Array {
+/** The biscuit key pair of an Identity: the same Ed25519 key under both libraries. */
+function biscuitKeyPair(identity: Identity): KeyPair {
+ return wasm.KeyPair.fromPrivateKey(wasm.PrivateKey.fromBytes(identity.toLibp2pPrivateKey().subarray(4, 36), wasm.SignatureAlgorithm.Ed25519));
+}
+
+function mint(peerId: string, role: string, expiration = "2035-01-01T00:00:00Z", labels: Record<string, string> = {}, signer: KeyPair = cpKeyPair): Uint8Array {
const b = wasm.Biscuit.builder();
b.addFact(wasm.Fact.fromString(`node(${JSON.stringify(peerId)})`));
b.addFact(wasm.Fact.fromString(`client_peer_id(${JSON.stringify(peerId)})`));
@@ -58,7 +70,7 @@ function mint(peerId: string, role: string, expiration = "2035-01-01T00:00:00Z",
for (const [k, v] of Object.entries(labels)) {
b.addFact(wasm.Fact.fromString(`label(${JSON.stringify(k)}, ${JSON.stringify(v)})`));
}
- return b.build(cpKeyPair.getPrivateKey()).toBytes();
+ return b.build(signer.getPrivateKey()).toBytes();
}
// What the control plane renders for a policy granting the node role every
@@ -110,8 +122,9 @@ function fakeControlPlane(routerAddresses: string[]): typeof fetch {
before(async () => {
wasm = await loadBiscuit();
- cpKeyPair = new wasm.KeyPair(wasm.SignatureAlgorithm.Ed25519);
- cpKey = new Uint8Array(Buffer.from(cpKeyPair.getPublicKey().toString().replace(/^ed25519\//, ""), "hex"));
+ cpKeyPair = biscuitKeyPair(cpIdentity);
+ cpKey = cpIdentity.publicKeyRaw;
+ assert.equal(Buffer.from(cpKey).toString("hex"), cpKeyPair.getPublicKey().toString().replace(/^ed25519\//, ""));
router = await createLibp2p({
addresses: { listen: ["/ip4/127.0.0.1/tcp/0"] },
@@ -375,6 +388,69 @@ test("a member on one router reaches a member on another", async () => {
}
});
+test("a resumed member adopts a rotated key before admitting routers", async () => {
+ // The control plane rotated its signing key while this member was not
+ // running: the routers already hold credentials under the new key, and
+ // the state directory holds only the key the member enrolled under. join
+ // pulls /keys, signed by the retiring key too, before it authenticates a
+ // router.
+ const rotatedIdentity = Identity.generate();
+ const rotated = biscuitKeyPair(rotatedIdentity);
+ const rotatedKey = rotatedIdentity.publicKeyRaw;
+ const controlPlaneAfterRotation: typeof fetch = (async (input: Parameters<typeof fetch>[0], init?: RequestInit) => {
+ const req = new Request(input, init);
+ const path = new URL(req.url).pathname;
+ if (req.method === "GET" && path === "/keys") {
+ const unsigned = create(KeysResponseSchema, { publicKeys: [cpKey, rotatedKey], signTime: timestampFromMs(Date.now()) });
+ const payload = toBinary(KeysResponseSchema, unsigned);
+ return proto(toBinary(KeysResponseSchema, create(KeysResponseSchema, { ...unsigned, signatures: [cpIdentity.sign(payload), rotatedIdentity.sign(payload)] })));
+ }
+ if (req.method === "POST" && path === "/refresh") {
+ const { peerId } = fromBinary(TokenRefreshRequestSchema, new Uint8Array(await req.arrayBuffer()));
+ return proto(toBinary(TokenRefreshResponseSchema, create(TokenRefreshResponseSchema, { biscuitToken: mint(peerId, ROLE_NODE, undefined, {}, rotated), expireTime: timestampFromMs(Date.now() + 7200_000) })));
+ }
+ return new Response(`no route for ${req.method} ${path}`, { status: 404 });
+ }) as typeof fetch;
+
+ const rotatedRouter = await createLibp2p({
+ addresses: { listen: ["/ip4/127.0.0.1/tcp/0"] },
+ transports: [tcp()],
+ connectionEncrypters: [tls()],
+ streamMuxers: [yamux()],
+ services: { identify: identify() },
+ });
+ const rotatedRouterBiscuit = mint(rotatedRouter.peerId.toString(), ROLE_ROUTER, undefined, {}, rotated);
+ await rotatedRouter.handle(AUTH_PROTOCOL, authStreamHandler({ ownBiscuit: () => rotatedRouterBiscuit, trustedKeys: () => [cpKey, rotatedKey] }));
+ const rotatedRouterAddr = (rotatedRouter.getMultiaddrs()[0] as ReturnType<typeof multiaddr>).toString();
+ const stateDir = await mkdtemp(join(tmpdir(), "sam-session-"));
+ try {
+ // Enrolled before the rotation: the credential on disk trusts one key.
+ const before = await AgentMesh.enroll({ controlPlaneUrl: "http://127.0.0.1:1", bootstrapToken: "sbt", stateDir, fetch: fakeControlPlane([rotatedRouterAddr]) });
+ assert.deepEqual(before.credential.controlPlaneKeys, [cpKey]);
+
+ // Resumed after it, with the routers now under the new key.
+ const resumed = await AgentMesh.enroll({ controlPlaneUrl: "http://127.0.0.1:1", stateDir, fetch: controlPlaneAfterRotation });
+ assert.deepEqual(resumed.credential.biscuit, before.credential.biscuit);
+ const session = await resumed.join({ reserveRelay: false, refreshLeadMs: 0 });
+ try {
+ assert.deepEqual(
+ session.routers.map((r) => r.peerId),
+ [rotatedRouter.peerId.toString()],
+ );
+ assert.deepEqual(
+ resumed.credential.controlPlaneKeys.map((k) => Buffer.from(k).toString("hex")).sort(),
+ [cpKey, rotatedKey].map((k) => Buffer.from(k).toString("hex")).sort(),
+ );
+ assert.notDeepEqual(resumed.credential.biscuit, before.credential.biscuit);
+ } finally {
+ await session.close();
+ }
+ } finally {
+ await rotatedRouter.stop();
+ await rm(stateDir, { recursive: true, force: true });
+ }
+});
+
test("join fails closed when the router is not a router", async () => {
// A relay whose credential lacks the router role must not admit us to the mesh.
const impostor = await createLibp2p({
diff --git a/sdk/js/src/session.ts b/sdk/js/src/session.ts
index e2aeccec..009a6044 100644
--- a/sdk/js/src/session.ts
+++ b/sdk/js/src/session.ts
@@ -750,6 +750,17 @@ export class MeshSession {
/** Implements AgentMesh.join(); lives here to keep mesh.ts free of libp2p. */
export async function joinMesh(mesh: AgentMesh, options: JoinOptions = {}): Promise<MeshSession> {
+ // The pull sam-node makes before it starts: a member resuming from its
+ // state directory after a key rotation would otherwise verify the routers,
+ // which already hold credentials under the new key, against the keys it
+ // persisted, and the sync that would have brought the new key runs only
+ // once joined. Best effort; the stored credential serves when the control
+ // plane cannot be reached.
+ try {
+ await mesh.syncControlPlane();
+ } catch {
+ // Joining goes on with what the credential holds.
+ }
const routerAddrs = (options.routerAddresses ?? mesh.credential.routerAddresses).map((a) => multiaddr(a));
if (routerAddrs.length === 0) {
throw new Error(options.routerAddresses === undefined ? "credential lists no router addresses; the control plane had no active router at enrollment" : "routerAddresses names no router");
diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.py
index eba20455..4902db4c 100644
--- a/sdk/python/src/agent_mesh/session.py
+++ b/sdk/python/src/agent_mesh/session.py
@@ -638,6 +638,18 @@ async def join_mesh(
credential lists; a peer behind another router is still reached, see
MeshSession.connect. egress_require_labels is the floor every provider
this member calls must attest, see MeshSession."""
+ # The pull sam-node makes before it starts: a member resuming from its
+ # state directory after a key rotation would otherwise verify the routers,
+ # which already hold credentials under the new key, against the keys it
+ # persisted, and the sync that would have brought the new key runs only
+ # once joined. Best effort; the stored credential serves when the control
+ # plane cannot be reached.
+ try:
+ result = await trio.to_thread.run_sync(mesh.sync_control_plane)
+ if result.errors:
+ logger.warning("control plane sync before join: %s", "; ".join(result.errors))
+ except Exception as err: # noqa: BLE001 - joining goes on with what the credential holds
+ logger.warning("control plane sync before join failed: %s", err)
router_addrs = [multiaddr.Multiaddr(a) for a in (mesh.credential.router_addresses if router_addresses is None else router_addresses)]
if not router_addrs:
raise RuntimeError("credential lists no router addresses; the control plane had no active router at enrollment" if router_addresses is None else "router_addresses names no router")
diff --git a/sdk/python/tests/test_session.py b/sdk/python/tests/test_session.py
index d69b41ee..2a72ca67 100644
--- a/sdk/python/tests/test_session.py
+++ b/sdk/python/tests/test_session.py
@@ -66,18 +66,21 @@ def _ts_s(seconds: int) -> Timestamp:
t.FromSeconds(int(seconds))
return t
-CP = ba.KeyPair()
+# The control plane's signing key, as an Identity so a test can sign /keys
+# with it and as the biscuit key pair that mints credentials.
+CP_IDENTITY = Identity.generate()
+CP = ba.KeyPair.from_private_key(ba.PrivateKey.from_bytes(CP_IDENTITY.seed, ba.Algorithm.Ed25519))
CP_KEY = CP.public_key.to_bytes()
-def mint(peer_id: str, role: str, expiration: str = "2035-01-01T00:00:00Z", labels: dict[str, str] | None = None) -> bytes:
+def mint(peer_id: str, role: str, expiration: str = "2035-01-01T00:00:00Z", labels: dict[str, str] | None = None, signer: ba.KeyPair = CP) -> bytes:
code = "node({p}); client_peer_id({p}); expiration(" + expiration + "); role({r});"
params = {"p": peer_id, "r": role}
for i, (k, v) in enumerate((labels or {}).items()):
code += f" label({{k{i}}}, {{v{i}}});"
params[f"k{i}"] = k
params[f"v{i}"] = v
- return ba.BiscuitBuilder(code, params).build(CP.private_key).to_bytes()
+ return ba.BiscuitBuilder(code, params).build(signer.private_key).to_bytes()
# What the control plane renders for a policy granting the node role every
@@ -150,11 +153,11 @@ def hop_handler(relay_addr: str, grants, ttl: int = 3600, events=None):
return handle
-async def start_router(nursery, role=ROLE_ROUTER, trusted=(CP_KEY,), grants=True, ttl=3600, events=None, identity=None):
+async def start_router(nursery, role=ROLE_ROUTER, trusted=(CP_KEY,), grants=True, ttl=3600, events=None, identity=None, signer=CP):
"""events, when given, records ("auth", peer) per handshake and ("reserve", peer) per RESERVE."""
identity = identity or Identity.generate()
router = libp2p_host(identity)
- router_biscuit = mint(identity.peer_id, role)
+ router_biscuit = mint(identity.peer_id, role, signer=signer)
def on_authenticated(peer, _verified):
if events is not None:
@@ -246,6 +249,54 @@ def test_join_authenticates_reserves_and_answers_peers():
trio.run(with_timeout, 30, main)
+def test_a_resumed_member_adopts_a_rotated_key_before_admitting_routers(tmp_path):
+ """The control plane rotated its signing key while this member was not
+ running: the routers already hold credentials under the new key, and the
+ state directory holds only the key the member enrolled under. join pulls
+ /keys, signed by the retiring key too, before it authenticates a router."""
+ rotated_identity = Identity.generate()
+ rotated = ba.KeyPair.from_private_key(ba.PrivateKey.from_bytes(rotated_identity.seed, ba.Algorithm.Ed25519))
+ rotated_key = rotated_identity.public_key_raw
+
+ def control_plane_after_rotation():
+ def transport(method, url, headers, body):
+ path = urllib.parse.urlsplit(url).path
+ if (method, path) == ("GET", "/keys"):
+ ts = _ts_ms(int(time.time() * 1000))
+ payload = pb.KeysResponse(public_keys=[CP_KEY, rotated_key], sign_time=ts).SerializeToString(deterministic=True)
+ return 200, pb.KeysResponse(
+ public_keys=[CP_KEY, rotated_key],
+ sign_time=ts,
+ signatures=[CP_IDENTITY.sign(payload), rotated_identity.sign(payload)],
+ ).SerializeToString()
+ if (method, path) == ("POST", "/refresh"):
+ peer_id = pb.TokenRefreshRequest.FromString(body).peer_id
+ return 200, pb.TokenRefreshResponse(
+ biscuit_token=mint(peer_id, ROLE_NODE, signer=rotated), expire_time=_ts_s(int(time.time()) + 7200)
+ ).SerializeToString()
+ return 404, f"no route for {method} {path}".encode()
+
+ return transport
+
+ async def main():
+ async with trio.open_nursery() as nursery:
+ router, router_addr = await start_router(nursery, trusted=(CP_KEY, rotated_key), signer=rotated)
+ # Enrolled before the rotation: the credential on disk trusts one key.
+ before = AgentMesh.enroll("http://127.0.0.1:1", bootstrap_token="sbt", state_dir=tmp_path, transport=fake_control_plane([router_addr]))
+ assert [bytes(k) for k in before.credential.control_plane_keys] == [CP_KEY]
+
+ # Resumed after it, with the routers now under the new key.
+ resumed = AgentMesh.enroll("http://127.0.0.1:1", state_dir=tmp_path, transport=control_plane_after_rotation())
+ assert resumed.credential.biscuit == before.credential.biscuit
+ async with resumed.join(reserve=False) as session:
+ assert [r.peer_id for r in session.routers] == [str(router.get_id())]
+ assert {bytes(k) for k in resumed.credential.control_plane_keys} == {CP_KEY, rotated_key}
+ assert resumed.credential.biscuit != before.credential.biscuit
+ nursery.cancel_scope.cancel()
+
+ trio.run(with_timeout, 30, main)
+
+
def test_join_fails_closed_when_the_router_is_not_a_router():
async def main():
async with trio.open_nursery() as nursery: