Skip to content

Agent Mesh

sdk: reserve again on a router where the control plane lists it now

Antonio Ojea049d7a23ed14
diff --git a/sdk/js/src/conformance-join.ts b/sdk/js/src/conformance-join.ts
index 54a12f6f..aa3fb217 100644
--- a/sdk/js/src/conformance-join.ts
+++ b/sdk/js/src/conformance-join.ts
@@ -196,6 +196,11 @@ async function main(): Promise<void> {
// is the floor every provider this member calls must attest.
const labels = labelsFromEnv("SAM_SDK_LABELS");
const egressRequireLabels = labelsFromEnv("SAM_SDK_EGRESS_REQUIRE_LABELS");
+ // SAM_SDK_RELAY_CHECK_SECONDS shortens how often the relay reservation is
+ // checked, so a test that moves a router sees the member follow it within
+ // its budget.
+ const relayCheck = process.env.SAM_SDK_RELAY_CHECK_SECONDS;
+ const relayOptions = relayCheck !== undefined && relayCheck !== "" ? { relayCheckIntervalMs: Number(relayCheck) * 1000 } : {};
const mesh = await AgentMesh.enroll({
controlPlaneUrl,
@@ -214,6 +219,7 @@ async function main(): Promise<void> {
listenAddrs,
...(routerAddresses !== undefined ? { routerAddresses } : {}),
...(Object.keys(egressRequireLabels).length > 0 ? { egressRequireLabels } : {}),
+ ...relayOptions,
signal: AbortSignal.timeout(20_000),
controlPlaneSyncIntervalMs: 0,
controlPlaneSyncJitterMs: 0,
diff --git a/sdk/js/src/session.test.ts b/sdk/js/src/session.test.ts
index 1b000131..6b5313ed 100644
--- a/sdk/js/src/session.test.ts
+++ b/sdk/js/src/session.test.ts
@@ -33,7 +33,7 @@ import { after, before, test } from "node:test";
import { AUTH_HANDLER_OPTIONS, AUTH_PROTOCOL, AuthRejectedError, authenticateWithPeer, authStreamHandler } from "./auth.ts";
import { BiscuitVerificationError, ROLE_ROUTER, loadBiscuit, verifyPeerBiscuit } from "./biscuit.ts";
import { ROLE_NODE } from "./controlplane.ts";
-import { BootstrapEnrollRequestSchema, BootstrapEnrollResponseSchema, EnrollmentStatus, KeysResponseSchema, AuthFrameSchema } from "./gen/sam_pb.ts";
+import { BootstrapEnrollRequestSchema, BootstrapEnrollResponseSchema, ControlPlaneInfoResponseSchema, EnrollmentStatus, KeysResponseSchema, AuthFrameSchema } from "./gen/sam_pb.ts";
import { Identity } from "./identity.ts";
import { HTTP_HANDLER_OPTIONS, HTTP_PROTOCOL, a2aEndpoint, httpIngressHandler } from "./libp2p-http.ts";
import { LabelsNotSatisfiedError } from "./mcp.ts";
@@ -73,7 +73,11 @@ function proto(bytes: Uint8Array): Response {
return new Response(Buffer.from(bytes), { status: 200, headers: { "Content-Type": "application/x-protobuf" } });
}
-/** Approves every enrollment with a biscuit bound to the requesting peer. */
+/**
+ * Approves every enrollment with a biscuit bound to the requesting peer.
+ * /info lists routerAddresses as they are at the time of the request, so a
+ * test that changes the array has the member pull the change.
+ */
function fakeControlPlane(routerAddresses: string[]): typeof fetch {
return (async (input: Parameters<typeof fetch>[0], init?: RequestInit) => {
const req = new Request(input, init);
@@ -97,6 +101,9 @@ function fakeControlPlane(routerAddresses: string[]): typeof fetch {
// Unsigned: the client keeps the enrollment key when /keys cannot be verified.
return proto(toBinary(KeysResponseSchema, create(KeysResponseSchema, { publicKeys: [cpKey], signTime: timestampFromMs(Date.now()) })));
}
+ if (req.method === "GET" && path === "/info") {
+ return proto(toBinary(ControlPlaneInfoResponseSchema, create(ControlPlaneInfoResponseSchema, { routerAddresses: [...routerAddresses] })));
+ }
return new Response(`no route for ${req.method} ${path}`, { status: 404 });
}) as typeof fetch;
}
@@ -270,6 +277,62 @@ test("a dropped router connection is authenticated and reserved again", async ()
}
});
+test("a router the control plane lists elsewhere is reserved on there", async () => {
+ // A router rescheduled keeps its key and comes back on another address. In
+ // a mesh that hands out literal addresses there is no name to re-resolve:
+ // the router is known only through the control plane's list, refreshed by
+ // every pull. The member must dial the address that list names now, not
+ // the one it was admitted on, and keep it for the circuits it opens.
+ const identity = Identity.generate();
+ const handshakes: { before: string[]; after: string[] } = { before: [], after: [] };
+ const startRouter = async (name: "before" | "after") => {
+ const node = await createLibp2p({
+ privateKey: privateKeyFromProtobuf(identity.toLibp2pPrivateKey()),
+ addresses: { listen: ["/ip4/127.0.0.1/tcp/0"] },
+ transports: [tcp()],
+ connectionEncrypters: [tls()],
+ streamMuxers: [yamux()],
+ services: { identify: identify(), relay: circuitRelayServer() },
+ });
+ const biscuit = mint(identity.peerId, ROLE_ROUTER);
+ await node.handle(AUTH_PROTOCOL, authStreamHandler({ ownBiscuit: () => biscuit, trustedKeys: () => [cpKey], onAuthenticated: (p) => handshakes[name].push(p) }));
+ return { node, addr: (node.getMultiaddrs()[0] as ReturnType<typeof multiaddr>).toString() };
+ };
+ const before = await startRouter("before");
+ const after = await startRouter("after");
+ try {
+ const listed = [before.addr];
+ const mesh = await AgentMesh.enroll({ controlPlaneUrl: "http://127.0.0.1:1", bootstrapToken: "sbt", fetch: fakeControlPlane(listed) });
+ const session = await mesh.join({ refreshLeadMs: 0, relayCheckIntervalMs: 200 });
+ try {
+ assert.deepEqual(session.routers.map((r) => r.addr.toString()), [before.addr]);
+ assert.deepEqual(handshakes, { before: [mesh.peerId], after: [] });
+
+ // The control plane lists the router where it is now; the member pulls that. Then the old pod goes.
+ listed.splice(0, listed.length, after.addr);
+ await session.sync();
+ assert.deepEqual(mesh.credential.routerAddresses, [after.addr]);
+ await before.node.stop();
+
+ const deadline = Date.now() + 10_000;
+ while (!session.relayAddresses.some((ma) => ma.toString().startsWith(after.addr)) && Date.now() < deadline) {
+ await new Promise((r) => setTimeout(r, 50));
+ }
+ assert.deepEqual(session.relayAddresses.map(String), [`${after.addr}/p2p-circuit/p2p/${mesh.peerId}`]);
+ // On the connection's open and ahead of the reservation, as after any drop; both the member.
+ assert.ok(handshakes.after.length >= 1 && handshakes.after.every((p) => p === mesh.peerId), `handshakes ${handshakes.after.join(",")}`);
+ assert.deepEqual(session.routers.map((r) => r.addr.toString()), [after.addr]);
+ const target = "12D3KooWA4Xop1JaT3MHxwYMkCepYsv4iPVopMXwCz5iHYdBfeSB";
+ assert.deepEqual(session.dialTargets(target).addrs.map(String), [`${after.addr}/p2p-circuit/p2p/${target}`]);
+ } finally {
+ await session.close();
+ }
+ } finally {
+ await after.node.stop();
+ await Promise.resolve(before.node.stop()).catch(() => {});
+ }
+});
+
test("a member on one router reaches a member on another", async () => {
// The caller joined through router A only and the agent reserved on router
// B only; the credential lists both, as the control plane's does. The
diff --git a/sdk/js/src/session.ts b/sdk/js/src/session.ts
index cae0437e..e2aeccec 100644
--- a/sdk/js/src/session.ts
+++ b/sdk/js/src/session.ts
@@ -249,9 +249,13 @@ export class MeshSession {
* at join: a router that restarted or trimmed the connection dropped it
* and with it the relayed address peers reach this member on. Routers are
* tried in order, each dialed and authenticated again first, since the
- * router forgot the admission with the connection. Runs on
- * relayCheckIntervalMs; exposed so a caller can force it. Concurrent
- * calls share one attempt. Upstream: libp2p/js-libp2p#3601.
+ * router forgot the admission with the connection. A router rescheduled
+ * keeps its key and comes back on another address; the list the control
+ * plane hands out, refreshed by every pull, names the current one, so the
+ * router is dialed at what that list says and keeps the address the
+ * connection was made on. Runs on relayCheckIntervalMs; exposed so a
+ * caller can force it. Concurrent calls share one attempt. Upstream:
+ * libp2p/js-libp2p#3601.
*/
keepRelay(): Promise<void> {
this.#keepingRelay ??= this.#keepRelayOnce().finally(() => {
@@ -265,11 +269,14 @@ export class MeshSession {
return;
}
const failures: string[] = [];
- for (const r of this.routers) {
+ for (const [i, r] of this.routers.entries()) {
+ const listed = this.mesh.credential.routerAddresses.map((a) => multiaddr(a)).filter((ma) => targetPeerOf(ma) === r.peerId);
try {
- const conn = await this.node.dial(r.addr);
+ const conn = await this.node.dial(listed.length > 0 ? listed : r.addr);
await authenticateWithPeer(conn, this.mesh.authFrame(), this.mesh.credential.controlPlaneKeys);
- await this.#relayListener.listen(r.addr.encapsulate("/p2p-circuit"));
+ const addr = connectedAddress(conn, r.peerId);
+ this.routers[i] = { ...r, addr };
+ await this.#relayListener.listen(addr.encapsulate("/p2p-circuit"));
return;
} catch (err) {
failures.push(`${r.peerId}: ${err instanceof Error ? err.message : String(err)}`);
diff --git a/sdk/python/src/agent_mesh/conformance_join.py b/sdk/python/src/agent_mesh/conformance_join.py
index 7e52dad1..aa7831e2 100644
--- a/sdk/python/src/agent_mesh/conformance_join.py
+++ b/sdk/python/src/agent_mesh/conformance_join.py
@@ -179,6 +179,11 @@ async def main() -> None:
# is the floor every provider this member calls must attest.
labels = _labels_from_env("SAM_SDK_LABELS")
egress_require_labels = _labels_from_env("SAM_SDK_EGRESS_REQUIRE_LABELS")
+ # SAM_SDK_RELAY_CHECK_SECONDS shortens how often the relay reservation is
+ # checked and how soon a failed renewal is retried, so a test that moves
+ # a router sees the member follow it within its budget.
+ relay_check = os.environ.get("SAM_SDK_RELAY_CHECK_SECONDS")
+ relay_options = {"reservation_check_interval": float(relay_check), "refresh_retry": float(relay_check)} if relay_check else {}
mesh = AgentMesh.enroll(
control_plane_url,
@@ -194,7 +199,12 @@ async def main() -> None:
only = {p for p in os.environ.get("SAM_SDK_ROUTERS", "").split(",") if p}
router_addresses = [a for a in mesh.credential.router_addresses if any(a.endswith(f"/p2p/{p}") for p in only)] if only else None
async with mesh.join(
- listen_addrs=listen, router_addresses=router_addresses, control_plane_sync_interval=0, control_plane_sync_jitter=0, egress_require_labels=egress_require_labels or None
+ listen_addrs=listen,
+ router_addresses=router_addresses,
+ control_plane_sync_interval=0,
+ control_plane_sync_jitter=0,
+ egress_require_labels=egress_require_labels or None,
+ **relay_options,
) as session:
_emit(
{
diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.py
index 4c8a1197..99ca3173 100644
--- a/sdk/python/src/agent_mesh/session.py
+++ b/sdk/python/src/agent_mesh/session.py
@@ -743,11 +743,19 @@ async def _authenticate_router(host: IHost, mesh: "AgentMesh", peer_id: ID) -> V
async def _reserve_again(host: IHost, mesh: "AgentMesh", router: AdmittedRouter) -> AdmittedRouter:
+ """Reserves on a router again, dialed and authenticated first when its
+ connection is gone. A router rescheduled keeps its key and comes back on
+ another address; the list the control plane hands out, refreshed by
+ every pull, names the current one, so the router is dialed at what that
+ list says and keeps that as its address. The address admitted at join
+ serves only when the list no longer names the router."""
peer_id = ID.from_base58(router.peer_id)
credential = router.credential
+ addr = router.addr
try:
if peer_id not in host.get_connected_peers():
- await dial(host, await peer_info(router.addr))
+ addr, info = await _current_router_info(mesh, router)
+ await dial(host, info)
credential = await _authenticate_router(host, mesh, peer_id)
reservation = await reserve_relay(host, peer_id)
except Exception:
@@ -758,4 +766,32 @@ async def _reserve_again(host: IHost, mesh: "AgentMesh", router: AdmittedRouter)
except Exception: # noqa: BLE001 - already gone
pass
raise
- return replace(router, credential=credential, reservation=reservation)
+ return replace(router, addr=addr, credential=credential, reservation=reservation)
+
+
+async def _current_router_info(mesh: "AgentMesh", router: AdmittedRouter) -> tuple[multiaddr.Multiaddr, PeerInfo]:
+ """The router's addresses as the credential lists them now, resolved to
+ what this host can dial, and the first of them this host can use as the
+ address to keep; the admitted address when the list has none for the
+ router."""
+ listed: list[multiaddr.Multiaddr] = []
+ for text in mesh.credential.router_addresses:
+ try:
+ ma = multiaddr.Multiaddr(text)
+ if str(info_from_p2p_addr(ma).peer_id) == router.peer_id:
+ listed.append(ma)
+ except Exception: # noqa: BLE001 - not a router address
+ continue
+ kept: Optional[multiaddr.Multiaddr] = None
+ dialable: list[multiaddr.Multiaddr] = []
+ failures: list[str] = []
+ for ma in listed or [router.addr]:
+ try:
+ dialable.extend(await dial_addrs(ma))
+ except Exception as err: # noqa: BLE001 - an address this host cannot use; the others are tried
+ failures.append(f"{ma}: {err}")
+ continue
+ kept = kept or ma
+ if kept is None:
+ raise RuntimeError("no address to dial router at:\n " + "\n ".join(failures))
+ return kept, PeerInfo(ID.from_base58(router.peer_id), dialable)
diff --git a/sdk/python/tests/test_session.py b/sdk/python/tests/test_session.py
index 174a24e8..d69b41ee 100644
--- a/sdk/python/tests/test_session.py
+++ b/sdk/python/tests/test_session.py
@@ -90,7 +90,9 @@ POLICY_RULES = [
def fake_control_plane(router_addresses):
- """Approves every enrollment with a biscuit bound to the requesting peer."""
+ """Approves every enrollment with a biscuit bound to the requesting peer.
+ /info lists router_addresses as they are at the time of the request, so
+ a test that changes the list has the member pull the change."""
def transport(method, url, headers, body):
path = urllib.parse.urlsplit(url).path
@@ -106,6 +108,8 @@ def fake_control_plane(router_addresses):
if (method, path) == ("GET", "/keys"):
# Unsigned: the client keeps the enrollment key when /keys cannot be verified.
return 200, pb.KeysResponse(public_keys=[CP_KEY], sign_time=_ts_ms(int(time.time() * 1000))).SerializeToString()
+ if (method, path) == ("GET", "/info"):
+ return 200, pb.ControlPlaneInfoResponse(router_addresses=list(router_addresses)).SerializeToString()
return 404, f"no route for {method} {path}".encode()
return transport
@@ -594,6 +598,49 @@ def test_a_router_that_came_back_on_another_address_is_reserved_on_again(monkeyp
trio.run(with_timeout, 60, main)
+def test_a_router_the_control_plane_lists_elsewhere_is_reserved_on_there():
+ """A mesh that hands out literal addresses (sam-one, a kind cluster) has
+ no name to re-resolve: a router that came back on another address is
+ known only through the control plane's list, refreshed by every pull.
+ The reservation loop dials the router at the address that list names
+ now and keeps it as the router's address, so the circuits connect()
+ opens through it go to the right place too."""
+
+ async def wait_for(predicate):
+ while not predicate():
+ await trio.sleep(0.1)
+
+ async def main():
+ async with trio.open_nursery() as nursery:
+ identity = Identity.generate()
+ before_events, after_events = [], []
+ before, before_addr = await start_router(nursery, events=before_events, identity=identity)
+ after, after_addr = await start_router(nursery, events=after_events, identity=identity)
+ listed = [before_addr]
+
+ mesh = AgentMesh.enroll("http://127.0.0.1:1", bootstrap_token="sbt", transport=fake_control_plane(listed))
+ async with mesh.join(refresh_lead=0, refresh_retry=0.5, reservation_check_interval=0.5) as session:
+ member = session.host.get_id()
+ assert [str(r.addr) for r in session.routers] == [before_addr]
+
+ # The control plane lists the router where it is now; the member pulls that.
+ listed[:] = [after_addr]
+ await session.sync()
+ assert list(mesh.credential.router_addresses) == [after_addr]
+ await before.disconnect(member)
+ await wait_for(lambda: member not in before.get_connected_peers())
+
+ reserve = ("reserve", mesh.peer_id)
+ await wait_for(lambda: before_events.count(reserve) + after_events.count(reserve) >= 2)
+ assert before_events.count(reserve) == 1, "dialed the address admitted at join again"
+ assert after_events == [("auth", mesh.peer_id), reserve]
+ assert [str(r.addr) for r in session.routers] == [after_addr]
+ assert session.relay_addresses == [f"{after_addr}/p2p-circuit/p2p/{mesh.peer_id}"]
+ nursery.cancel_scope.cancel()
+
+ trio.run(with_timeout, 60, main)
+
+
def test_join_renews_the_reservation_and_reauthenticates_after_a_disconnect():
"""go-libp2p's relay drops a reservation when its TTL passes and grants one
only to a peer authenticated on the current connection. The member renews
diff --git a/tests/integration/sdk_mesh_test.go b/tests/integration/sdk_mesh_test.go
index 51c94f05..e1abef80 100644
--- a/tests/integration/sdk_mesh_test.go
+++ b/tests/integration/sdk_mesh_test.go
@@ -871,6 +871,85 @@ func TestNativeSDKsAcrossRouters(t *testing.T) {
}
}
+// A router rescheduled keeps its key and comes back on another address. A
+// member that reserved on it advertises a dead relayed address until it
+// reserves again, and it must do so at the router's new address: here a
+// literal one, as sam-one hands out, so there is no name to re-resolve and
+// only the control plane's list, which the member pulls, names it.
+func TestNativeSDKsFollowAMovedRouter(t *testing.T) {
+ mesh := startSDKMesh(t)
+ keys := t.TempDir()
+ routerAddr, stop := startRouter(t, keys, mesh.cpPort, mesh.mintToken, "router-c")
+ router := extractPeerID(routerAddr)
+
+ var members []*sdkMember
+ for _, launcher := range sdkMemberLaunchers {
+ cmd, skip := launcher.cmd(mesh.root)
+ if skip != "" {
+ t.Logf("%s SDK skipped: %s", launcher.name, skip)
+ continue
+ }
+ m := launchSDKMember(t, launcher.name, cmd, mesh.root, mesh.baseURL, mesh.adminToken, "SAM_SDK_ROUTERS="+router, "SAM_SDK_RELAY_CHECK_SECONDS=0.5")
+ if got := routerIDs(m.report); !reflect.DeepEqual(got, []string{router}) {
+ t.Fatalf("%s joined through %v, want %s only", m.name, got, router)
+ }
+ for _, a := range m.report.RelayAddresses {
+ if !strings.HasPrefix(a, routerAddr) {
+ t.Fatalf("%s reserved on %s, want %s", m.name, a, routerAddr)
+ }
+ }
+ m.accept(t, "agent")
+ members = append(members, m)
+ }
+ if len(members) == 0 {
+ t.Skip("no SDK toolchain available; see sdk/README.md")
+ }
+
+ // The same key on another port; the router's lease replaces its address.
+ stop()
+ movedAddr, _ := startRouter(t, keys, mesh.cpPort, mesh.mintToken, "router-c")
+ if extractPeerID(movedAddr) != router || movedAddr == routerAddr {
+ t.Fatalf("router came back as %s, want %s on another port", movedAddr, routerAddr)
+ }
+
+ for _, m := range members {
+ if got := m.sync(t).RouterAddresses; !contains(got, movedAddr) || contains(got, routerAddr) {
+ t.Fatalf("%s pulled routers %v, want %s and not %s", m.name, got, movedAddr, routerAddr)
+ }
+ deadline := time.Now().Add(15 * time.Second)
+ for {
+ relayed := m.relayAddresses(t)
+ if len(relayed) > 0 && strings.HasPrefix(relayed[0], movedAddr) {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("%s still advertises %v, want the router at %s", m.name, relayed, movedAddr)
+ }
+ time.Sleep(200 * time.Millisecond)
+ }
+ }
+
+ // A peer that knows the router where it is now reaches each member there.
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ goPeer := newAdmittedGoPeer(t, ctx, mesh.cpPriv, []string{movedAddr})
+ for _, m := range members {
+ sdkPeer, err := peer.Decode(m.report.PeerID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := goPeer.Connect(ctx, peer.AddrInfo{ID: sdkPeer, Addrs: []multiaddr.Multiaddr{multiaddr.StringCast(m.relayAddresses(t)[0])}}); err != nil {
+ t.Fatalf("go peer could not reach %s through the moved router: %v", m.name, err)
+ }
+ if status, body := libp2pHTTPGet(t, ctx, goPeer, sdkPeer, goHostBiscuit(t, mesh.cpPriv, goPeer.ID()), "/a2a/agent/card"); status != 200 || !strings.Contains(body, goPeer.ID().String()) {
+ t.Fatalf("%s answered %d %s through the moved router", m.name, status, body)
+ }
+ }
+ for _, m := range members {
+ m.quit(t)
+ }
+}
+
// routerIDs lists the routers a join report says admitted the member.
func routerIDs(report sdkJoinReport) []string {
var ids []string
@@ -1242,18 +1321,30 @@ func (m *sdkMember) http(t *testing.T, addr, service, path string) sdkHTTPResult
// routers asks the member which routers admitted it so far.
func (m *sdkMember) routers(t *testing.T) []string {
t.Helper()
- var res struct {
- OK bool `json:"ok"`
- Error string `json:"error"`
- Routers []string `json:"routers"`
- }
+ return m.routersAnswer(t).Routers
+}
+
+// relayAddresses are the `.../p2p-circuit/p2p/<member>` addresses the member
+// holds a reservation for right now.
+func (m *sdkMember) relayAddresses(t *testing.T) []string {
+ t.Helper()
+ return m.routersAnswer(t).RelayAddresses
+}
+
+func (m *sdkMember) routersAnswer(t *testing.T) (res struct {
+ OK bool `json:"ok"`
+ Error string `json:"error"`
+ Routers []string `json:"routers"`
+ RelayAddresses []string `json:"relay_addresses"`
+}) {
+ t.Helper()
if line := m.send(t, map[string]string{"cmd": "routers"}); json.Unmarshal(line, &res) != nil {
t.Fatalf("%s member: routers answered %q", m.name, line)
}
if !res.OK {
t.Fatalf("%s member: routers: %s", m.name, res.Error)
}
- return res.Routers
+ return res
}
func (m *sdkMember) quit(t *testing.T) {