Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
sdk: reserve again on a router where the control plane lists it now
Antonio Ojea049d7a23ed14
diff --git a/sdk/js/src/conformance-join.ts b/sdk/js/src/conformance-join.tsindex 54a12f6f..aa3fb217 100644--- a/sdk/js/src/conformance-join.ts+++ b/sdk/js/src/conformance-join.ts@@ -196,6 +196,11 @@ async function main(): Promise<void> {// is the floor every provider this member calls must attest.const labels = labelsFromEnv("SAM_SDK_LABELS");const egressRequireLabels = labelsFromEnv("SAM_SDK_EGRESS_REQUIRE_LABELS");+ // SAM_SDK_RELAY_CHECK_SECONDS shortens how often the relay reservation is+ // checked, so a test that moves a router sees the member follow it within+ // its budget.+ const relayCheck = process.env.SAM_SDK_RELAY_CHECK_SECONDS;+ const relayOptions = relayCheck !== undefined && relayCheck !== "" ? { relayCheckIntervalMs: Number(relayCheck) * 1000 } : {};const mesh = await AgentMesh.enroll({controlPlaneUrl,@@ -214,6 +219,7 @@ async function main(): Promise<void> {listenAddrs,...(routerAddresses !== undefined ? { routerAddresses } : {}),...(Object.keys(egressRequireLabels).length > 0 ? { egressRequireLabels } : {}),+ ...relayOptions,signal: AbortSignal.timeout(20_000),controlPlaneSyncIntervalMs: 0,controlPlaneSyncJitterMs: 0,diff --git a/sdk/js/src/session.test.ts b/sdk/js/src/session.test.tsindex 1b000131..6b5313ed 100644--- a/sdk/js/src/session.test.ts+++ b/sdk/js/src/session.test.ts@@ -33,7 +33,7 @@ import { after, before, test } from "node:test";import { AUTH_HANDLER_OPTIONS, AUTH_PROTOCOL, AuthRejectedError, authenticateWithPeer, authStreamHandler } from "./auth.ts";import { BiscuitVerificationError, ROLE_ROUTER, loadBiscuit, verifyPeerBiscuit } from "./biscuit.ts";import { ROLE_NODE } from "./controlplane.ts";-import { BootstrapEnrollRequestSchema, BootstrapEnrollResponseSchema, EnrollmentStatus, KeysResponseSchema, AuthFrameSchema } from "./gen/sam_pb.ts";+import { BootstrapEnrollRequestSchema, BootstrapEnrollResponseSchema, ControlPlaneInfoResponseSchema, EnrollmentStatus, KeysResponseSchema, AuthFrameSchema } from "./gen/sam_pb.ts";import { Identity } from "./identity.ts";import { HTTP_HANDLER_OPTIONS, HTTP_PROTOCOL, a2aEndpoint, httpIngressHandler } from "./libp2p-http.ts";import { LabelsNotSatisfiedError } from "./mcp.ts";@@ -73,7 +73,11 @@ function proto(bytes: Uint8Array): Response {return new Response(Buffer.from(bytes), { status: 200, headers: { "Content-Type": "application/x-protobuf" } });}-/** Approves every enrollment with a biscuit bound to the requesting peer. */+/**+ * Approves every enrollment with a biscuit bound to the requesting peer.+ * /info lists routerAddresses as they are at the time of the request, so a+ * test that changes the array has the member pull the change.+ */function fakeControlPlane(routerAddresses: string[]): typeof fetch {return (async (input: Parameters<typeof fetch>[0], init?: RequestInit) => {const req = new Request(input, init);@@ -97,6 +101,9 @@ function fakeControlPlane(routerAddresses: string[]): typeof fetch {// Unsigned: the client keeps the enrollment key when /keys cannot be verified.return proto(toBinary(KeysResponseSchema, create(KeysResponseSchema, { publicKeys: [cpKey], signTime: timestampFromMs(Date.now()) })));}+ if (req.method === "GET" && path === "/info") {+ return proto(toBinary(ControlPlaneInfoResponseSchema, create(ControlPlaneInfoResponseSchema, { routerAddresses: [...routerAddresses] })));+ }return new Response(`no route for ${req.method} ${path}`, { status: 404 });}) as typeof fetch;}@@ -270,6 +277,62 @@ test("a dropped router connection is authenticated and reserved again", async ()}});+test("a router the control plane lists elsewhere is reserved on there", async () => {+ // A router rescheduled keeps its key and comes back on another address. In+ // a mesh that hands out literal addresses there is no name to re-resolve:+ // the router is known only through the control plane's list, refreshed by+ // every pull. The member must dial the address that list names now, not+ // the one it was admitted on, and keep it for the circuits it opens.+ const identity = Identity.generate();+ const handshakes: { before: string[]; after: string[] } = { before: [], after: [] };+ const startRouter = async (name: "before" | "after") => {+ const node = await createLibp2p({+ privateKey: privateKeyFromProtobuf(identity.toLibp2pPrivateKey()),+ addresses: { listen: ["/ip4/127.0.0.1/tcp/0"] },+ transports: [tcp()],+ connectionEncrypters: [tls()],+ streamMuxers: [yamux()],+ services: { identify: identify(), relay: circuitRelayServer() },+ });+ const biscuit = mint(identity.peerId, ROLE_ROUTER);+ await node.handle(AUTH_PROTOCOL, authStreamHandler({ ownBiscuit: () => biscuit, trustedKeys: () => [cpKey], onAuthenticated: (p) => handshakes[name].push(p) }));+ return { node, addr: (node.getMultiaddrs()[0] as ReturnType<typeof multiaddr>).toString() };+ };+ const before = await startRouter("before");+ const after = await startRouter("after");+ try {+ const listed = [before.addr];+ const mesh = await AgentMesh.enroll({ controlPlaneUrl: "http://127.0.0.1:1", bootstrapToken: "sbt", fetch: fakeControlPlane(listed) });+ const session = await mesh.join({ refreshLeadMs: 0, relayCheckIntervalMs: 200 });+ try {+ assert.deepEqual(session.routers.map((r) => r.addr.toString()), [before.addr]);+ assert.deepEqual(handshakes, { before: [mesh.peerId], after: [] });++ // The control plane lists the router where it is now; the member pulls that. Then the old pod goes.+ listed.splice(0, listed.length, after.addr);+ await session.sync();+ assert.deepEqual(mesh.credential.routerAddresses, [after.addr]);+ await before.node.stop();++ const deadline = Date.now() + 10_000;+ while (!session.relayAddresses.some((ma) => ma.toString().startsWith(after.addr)) && Date.now() < deadline) {+ await new Promise((r) => setTimeout(r, 50));+ }+ assert.deepEqual(session.relayAddresses.map(String), [`${after.addr}/p2p-circuit/p2p/${mesh.peerId}`]);+ // On the connection's open and ahead of the reservation, as after any drop; both the member.+ assert.ok(handshakes.after.length >= 1 && handshakes.after.every((p) => p === mesh.peerId), `handshakes ${handshakes.after.join(",")}`);+ assert.deepEqual(session.routers.map((r) => r.addr.toString()), [after.addr]);+ const target = "12D3KooWA4Xop1JaT3MHxwYMkCepYsv4iPVopMXwCz5iHYdBfeSB";+ assert.deepEqual(session.dialTargets(target).addrs.map(String), [`${after.addr}/p2p-circuit/p2p/${target}`]);+ } finally {+ await session.close();+ }+ } finally {+ await after.node.stop();+ await Promise.resolve(before.node.stop()).catch(() => {});+ }+});+test("a member on one router reaches a member on another", async () => {// The caller joined through router A only and the agent reserved on router// B only; the credential lists both, as the control plane's does. Thediff --git a/sdk/js/src/session.ts b/sdk/js/src/session.tsindex cae0437e..e2aeccec 100644--- a/sdk/js/src/session.ts+++ b/sdk/js/src/session.ts@@ -249,9 +249,13 @@ export class MeshSession {* at join: a router that restarted or trimmed the connection dropped it* and with it the relayed address peers reach this member on. Routers are* tried in order, each dialed and authenticated again first, since the- * router forgot the admission with the connection. Runs on- * relayCheckIntervalMs; exposed so a caller can force it. Concurrent- * calls share one attempt. Upstream: libp2p/js-libp2p#3601.+ * router forgot the admission with the connection. A router rescheduled+ * keeps its key and comes back on another address; the list the control+ * plane hands out, refreshed by every pull, names the current one, so the+ * router is dialed at what that list says and keeps the address the+ * connection was made on. Runs on relayCheckIntervalMs; exposed so a+ * caller can force it. Concurrent calls share one attempt. Upstream:+ * libp2p/js-libp2p#3601.*/keepRelay(): Promise<void> {this.#keepingRelay ??= this.#keepRelayOnce().finally(() => {@@ -265,11 +269,14 @@ export class MeshSession {return;}const failures: string[] = [];- for (const r of this.routers) {+ for (const [i, r] of this.routers.entries()) {+ const listed = this.mesh.credential.routerAddresses.map((a) => multiaddr(a)).filter((ma) => targetPeerOf(ma) === r.peerId);try {- const conn = await this.node.dial(r.addr);+ const conn = await this.node.dial(listed.length > 0 ? listed : r.addr);await authenticateWithPeer(conn, this.mesh.authFrame(), this.mesh.credential.controlPlaneKeys);- await this.#relayListener.listen(r.addr.encapsulate("/p2p-circuit"));+ const addr = connectedAddress(conn, r.peerId);+ this.routers[i] = { ...r, addr };+ await this.#relayListener.listen(addr.encapsulate("/p2p-circuit"));return;} catch (err) {failures.push(`${r.peerId}: ${err instanceof Error ? err.message : String(err)}`);diff --git a/sdk/python/src/agent_mesh/conformance_join.py b/sdk/python/src/agent_mesh/conformance_join.pyindex 7e52dad1..aa7831e2 100644--- a/sdk/python/src/agent_mesh/conformance_join.py+++ b/sdk/python/src/agent_mesh/conformance_join.py@@ -179,6 +179,11 @@ async def main() -> None:# is the floor every provider this member calls must attest.labels = _labels_from_env("SAM_SDK_LABELS")egress_require_labels = _labels_from_env("SAM_SDK_EGRESS_REQUIRE_LABELS")+ # SAM_SDK_RELAY_CHECK_SECONDS shortens how often the relay reservation is+ # checked and how soon a failed renewal is retried, so a test that moves+ # a router sees the member follow it within its budget.+ relay_check = os.environ.get("SAM_SDK_RELAY_CHECK_SECONDS")+ relay_options = {"reservation_check_interval": float(relay_check), "refresh_retry": float(relay_check)} if relay_check else {}mesh = AgentMesh.enroll(control_plane_url,@@ -194,7 +199,12 @@ async def main() -> None:only = {p for p in os.environ.get("SAM_SDK_ROUTERS", "").split(",") if p}router_addresses = [a for a in mesh.credential.router_addresses if any(a.endswith(f"/p2p/{p}") for p in only)] if only else Noneasync with mesh.join(- listen_addrs=listen, router_addresses=router_addresses, control_plane_sync_interval=0, control_plane_sync_jitter=0, egress_require_labels=egress_require_labels or None+ listen_addrs=listen,+ router_addresses=router_addresses,+ control_plane_sync_interval=0,+ control_plane_sync_jitter=0,+ egress_require_labels=egress_require_labels or None,+ **relay_options,) as session:_emit({diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.pyindex 4c8a1197..99ca3173 100644--- a/sdk/python/src/agent_mesh/session.py+++ b/sdk/python/src/agent_mesh/session.py@@ -743,11 +743,19 @@ async def _authenticate_router(host: IHost, mesh: "AgentMesh", peer_id: ID) -> Vasync def _reserve_again(host: IHost, mesh: "AgentMesh", router: AdmittedRouter) -> AdmittedRouter:+ """Reserves on a router again, dialed and authenticated first when its+ connection is gone. A router rescheduled keeps its key and comes back on+ another address; the list the control plane hands out, refreshed by+ every pull, names the current one, so the router is dialed at what that+ list says and keeps that as its address. The address admitted at join+ serves only when the list no longer names the router."""peer_id = ID.from_base58(router.peer_id)credential = router.credential+ addr = router.addrtry:if peer_id not in host.get_connected_peers():- await dial(host, await peer_info(router.addr))+ addr, info = await _current_router_info(mesh, router)+ await dial(host, info)credential = await _authenticate_router(host, mesh, peer_id)reservation = await reserve_relay(host, peer_id)except Exception:@@ -758,4 +766,32 @@ async def _reserve_again(host: IHost, mesh: "AgentMesh", router: AdmittedRouter)except Exception: # noqa: BLE001 - already gonepassraise- return replace(router, credential=credential, reservation=reservation)+ return replace(router, addr=addr, credential=credential, reservation=reservation)+++async def _current_router_info(mesh: "AgentMesh", router: AdmittedRouter) -> tuple[multiaddr.Multiaddr, PeerInfo]:+ """The router's addresses as the credential lists them now, resolved to+ what this host can dial, and the first of them this host can use as the+ address to keep; the admitted address when the list has none for the+ router."""+ listed: list[multiaddr.Multiaddr] = []+ for text in mesh.credential.router_addresses:+ try:+ ma = multiaddr.Multiaddr(text)+ if str(info_from_p2p_addr(ma).peer_id) == router.peer_id:+ listed.append(ma)+ except Exception: # noqa: BLE001 - not a router address+ continue+ kept: Optional[multiaddr.Multiaddr] = None+ dialable: list[multiaddr.Multiaddr] = []+ failures: list[str] = []+ for ma in listed or [router.addr]:+ try:+ dialable.extend(await dial_addrs(ma))+ except Exception as err: # noqa: BLE001 - an address this host cannot use; the others are tried+ failures.append(f"{ma}: {err}")+ continue+ kept = kept or ma+ if kept is None:+ raise RuntimeError("no address to dial router at:\n " + "\n ".join(failures))+ return kept, PeerInfo(ID.from_base58(router.peer_id), dialable)diff --git a/sdk/python/tests/test_session.py b/sdk/python/tests/test_session.pyindex 174a24e8..d69b41ee 100644--- a/sdk/python/tests/test_session.py+++ b/sdk/python/tests/test_session.py@@ -90,7 +90,9 @@ POLICY_RULES = [def fake_control_plane(router_addresses):- """Approves every enrollment with a biscuit bound to the requesting peer."""+ """Approves every enrollment with a biscuit bound to the requesting peer.+ /info lists router_addresses as they are at the time of the request, so+ a test that changes the list has the member pull the change."""def transport(method, url, headers, body):path = urllib.parse.urlsplit(url).path@@ -106,6 +108,8 @@ def fake_control_plane(router_addresses):if (method, path) == ("GET", "/keys"):# Unsigned: the client keeps the enrollment key when /keys cannot be verified.return 200, pb.KeysResponse(public_keys=[CP_KEY], sign_time=_ts_ms(int(time.time() * 1000))).SerializeToString()+ if (method, path) == ("GET", "/info"):+ return 200, pb.ControlPlaneInfoResponse(router_addresses=list(router_addresses)).SerializeToString()return 404, f"no route for {method} {path}".encode()return transport@@ -594,6 +598,49 @@ def test_a_router_that_came_back_on_another_address_is_reserved_on_again(monkeyptrio.run(with_timeout, 60, main)+def test_a_router_the_control_plane_lists_elsewhere_is_reserved_on_there():+ """A mesh that hands out literal addresses (sam-one, a kind cluster) has+ no name to re-resolve: a router that came back on another address is+ known only through the control plane's list, refreshed by every pull.+ The reservation loop dials the router at the address that list names+ now and keeps it as the router's address, so the circuits connect()+ opens through it go to the right place too."""++ async def wait_for(predicate):+ while not predicate():+ await trio.sleep(0.1)++ async def main():+ async with trio.open_nursery() as nursery:+ identity = Identity.generate()+ before_events, after_events = [], []+ before, before_addr = await start_router(nursery, events=before_events, identity=identity)+ after, after_addr = await start_router(nursery, events=after_events, identity=identity)+ listed = [before_addr]++ mesh = AgentMesh.enroll("http://127.0.0.1:1", bootstrap_token="sbt", transport=fake_control_plane(listed))+ async with mesh.join(refresh_lead=0, refresh_retry=0.5, reservation_check_interval=0.5) as session:+ member = session.host.get_id()+ assert [str(r.addr) for r in session.routers] == [before_addr]++ # The control plane lists the router where it is now; the member pulls that.+ listed[:] = [after_addr]+ await session.sync()+ assert list(mesh.credential.router_addresses) == [after_addr]+ await before.disconnect(member)+ await wait_for(lambda: member not in before.get_connected_peers())++ reserve = ("reserve", mesh.peer_id)+ await wait_for(lambda: before_events.count(reserve) + after_events.count(reserve) >= 2)+ assert before_events.count(reserve) == 1, "dialed the address admitted at join again"+ assert after_events == [("auth", mesh.peer_id), reserve]+ assert [str(r.addr) for r in session.routers] == [after_addr]+ assert session.relay_addresses == [f"{after_addr}/p2p-circuit/p2p/{mesh.peer_id}"]+ nursery.cancel_scope.cancel()++ trio.run(with_timeout, 60, main)++def test_join_renews_the_reservation_and_reauthenticates_after_a_disconnect():"""go-libp2p's relay drops a reservation when its TTL passes and grants oneonly to a peer authenticated on the current connection. The member renewsdiff --git a/tests/integration/sdk_mesh_test.go b/tests/integration/sdk_mesh_test.goindex 51c94f05..e1abef80 100644--- a/tests/integration/sdk_mesh_test.go+++ b/tests/integration/sdk_mesh_test.go@@ -871,6 +871,85 @@ func TestNativeSDKsAcrossRouters(t *testing.T) {}}+// A router rescheduled keeps its key and comes back on another address. A+// member that reserved on it advertises a dead relayed address until it+// reserves again, and it must do so at the router's new address: here a+// literal one, as sam-one hands out, so there is no name to re-resolve and+// only the control plane's list, which the member pulls, names it.+func TestNativeSDKsFollowAMovedRouter(t *testing.T) {+ mesh := startSDKMesh(t)+ keys := t.TempDir()+ routerAddr, stop := startRouter(t, keys, mesh.cpPort, mesh.mintToken, "router-c")+ router := extractPeerID(routerAddr)++ var members []*sdkMember+ for _, launcher := range sdkMemberLaunchers {+ cmd, skip := launcher.cmd(mesh.root)+ if skip != "" {+ t.Logf("%s SDK skipped: %s", launcher.name, skip)+ continue+ }+ m := launchSDKMember(t, launcher.name, cmd, mesh.root, mesh.baseURL, mesh.adminToken, "SAM_SDK_ROUTERS="+router, "SAM_SDK_RELAY_CHECK_SECONDS=0.5")+ if got := routerIDs(m.report); !reflect.DeepEqual(got, []string{router}) {+ t.Fatalf("%s joined through %v, want %s only", m.name, got, router)+ }+ for _, a := range m.report.RelayAddresses {+ if !strings.HasPrefix(a, routerAddr) {+ t.Fatalf("%s reserved on %s, want %s", m.name, a, routerAddr)+ }+ }+ m.accept(t, "agent")+ members = append(members, m)+ }+ if len(members) == 0 {+ t.Skip("no SDK toolchain available; see sdk/README.md")+ }++ // The same key on another port; the router's lease replaces its address.+ stop()+ movedAddr, _ := startRouter(t, keys, mesh.cpPort, mesh.mintToken, "router-c")+ if extractPeerID(movedAddr) != router || movedAddr == routerAddr {+ t.Fatalf("router came back as %s, want %s on another port", movedAddr, routerAddr)+ }++ for _, m := range members {+ if got := m.sync(t).RouterAddresses; !contains(got, movedAddr) || contains(got, routerAddr) {+ t.Fatalf("%s pulled routers %v, want %s and not %s", m.name, got, movedAddr, routerAddr)+ }+ deadline := time.Now().Add(15 * time.Second)+ for {+ relayed := m.relayAddresses(t)+ if len(relayed) > 0 && strings.HasPrefix(relayed[0], movedAddr) {+ break+ }+ if time.Now().After(deadline) {+ t.Fatalf("%s still advertises %v, want the router at %s", m.name, relayed, movedAddr)+ }+ time.Sleep(200 * time.Millisecond)+ }+ }++ // A peer that knows the router where it is now reaches each member there.+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()+ goPeer := newAdmittedGoPeer(t, ctx, mesh.cpPriv, []string{movedAddr})+ for _, m := range members {+ sdkPeer, err := peer.Decode(m.report.PeerID)+ if err != nil {+ t.Fatal(err)+ }+ if err := goPeer.Connect(ctx, peer.AddrInfo{ID: sdkPeer, Addrs: []multiaddr.Multiaddr{multiaddr.StringCast(m.relayAddresses(t)[0])}}); err != nil {+ t.Fatalf("go peer could not reach %s through the moved router: %v", m.name, err)+ }+ if status, body := libp2pHTTPGet(t, ctx, goPeer, sdkPeer, goHostBiscuit(t, mesh.cpPriv, goPeer.ID()), "/a2a/agent/card"); status != 200 || !strings.Contains(body, goPeer.ID().String()) {+ t.Fatalf("%s answered %d %s through the moved router", m.name, status, body)+ }+ }+ for _, m := range members {+ m.quit(t)+ }+}+// routerIDs lists the routers a join report says admitted the member.func routerIDs(report sdkJoinReport) []string {var ids []string@@ -1242,18 +1321,30 @@ func (m *sdkMember) http(t *testing.T, addr, service, path string) sdkHTTPResult// routers asks the member which routers admitted it so far.func (m *sdkMember) routers(t *testing.T) []string {t.Helper()- var res struct {- OK bool `json:"ok"`- Error string `json:"error"`- Routers []string `json:"routers"`- }+ return m.routersAnswer(t).Routers+}++// relayAddresses are the `.../p2p-circuit/p2p/<member>` addresses the member+// holds a reservation for right now.+func (m *sdkMember) relayAddresses(t *testing.T) []string {+ t.Helper()+ return m.routersAnswer(t).RelayAddresses+}++func (m *sdkMember) routersAnswer(t *testing.T) (res struct {+ OK bool `json:"ok"`+ Error string `json:"error"`+ Routers []string `json:"routers"`+ RelayAddresses []string `json:"relay_addresses"`+}) {+ t.Helper()if line := m.send(t, map[string]string{"cmd": "routers"}); json.Unmarshal(line, &res) != nil {t.Fatalf("%s member: routers answered %q", m.name, line)}if !res.OK {t.Fatalf("%s member: routers: %s", m.name, res.Error)}- return res.Routers+ return res}func (m *sdkMember) quit(t *testing.T) {