Skip to content

Agent Mesh

sdk: a member on one router reaches a member on another

Antonio Ojea790e06b4f8bc
diff --git a/sdk/js/src/conformance-join.ts b/sdk/js/src/conformance-join.ts
index f5fd28a9..e00d4f0f 100644
--- a/sdk/js/src/conformance-join.ts
+++ b/sdk/js/src/conformance-join.ts
@@ -117,6 +117,8 @@ async function handle(session: MeshSession, command: Command): Promise<unknown>
}
case "peers":
return { cmd: "peers", authenticated_peers: [...session.authenticatedPeers.keys()].sort() };
+ case "routers":
+ return { cmd: "routers", ok: true, routers: session.routers.map((r) => r.peerId), relay_addresses: session.relayAddresses.map((ma) => ma.toString()) };
case "sync": {
const result = await session.sync();
return {
@@ -188,7 +190,17 @@ async function main(): Promise<void> {
...(Object.keys(labels).length > 0 ? { labels } : {}),
});
// The test drives every pull itself; only gossip events bring one forward.
- const session = await mesh.join({ listenAddrs, signal: AbortSignal.timeout(20_000), controlPlaneSyncIntervalMs: 0, controlPlaneSyncJitterMs: 0 });
+ // SAM_SDK_ROUTERS, peer IDs, joins through those routers only, so the test
+ // can put two members on different routers.
+ const only = new Set((process.env.SAM_SDK_ROUTERS ?? "").split(",").filter((id) => id !== ""));
+ const routerAddresses = only.size === 0 ? undefined : mesh.credential.routerAddresses.filter((a) => [...only].some((id) => a.endsWith(`/p2p/${id}`)));
+ const session = await mesh.join({
+ listenAddrs,
+ ...(routerAddresses !== undefined ? { routerAddresses } : {}),
+ signal: AbortSignal.timeout(20_000),
+ controlPlaneSyncIntervalMs: 0,
+ controlPlaneSyncJitterMs: 0,
+ });
emit({
sdk: "js",
diff --git a/sdk/js/src/session.test.ts b/sdk/js/src/session.test.ts
index e2f53415..6829ac42 100644
--- a/sdk/js/src/session.test.ts
+++ b/sdk/js/src/session.test.ts
@@ -253,6 +253,48 @@ test("a dropped router connection is authenticated and reserved again", async ()
}
});
+test("a member on one router reaches a member on another", async () => {
+ // The caller joined through router A only and the agent reserved on router
+ // B only; the credential lists both, as the control plane's does. The
+ // caller must not depend on having joined through the agent's router:
+ // it admits B on the way and dials the circuit.
+ const routerB = await createLibp2p({
+ addresses: { listen: ["/ip4/127.0.0.1/tcp/0"] },
+ transports: [tcp()],
+ connectionEncrypters: [tls()],
+ streamMuxers: [yamux()],
+ services: { identify: identify(), relay: circuitRelayServer() },
+ });
+ try {
+ const routerBBiscuit = mint(routerB.peerId.toString(), ROLE_ROUTER);
+ const handshakesB: string[] = [];
+ await routerB.handle(AUTH_PROTOCOL, authStreamHandler({ ownBiscuit: () => routerBBiscuit, trustedKeys: () => [cpKey], onAuthenticated: (p) => handshakesB.push(p) }));
+ const routerBAddr = (routerB.getMultiaddrs()[0] as ReturnType<typeof multiaddr>).toString();
+ const both = [routerAddr, routerBAddr];
+
+ const agent = await AgentMesh.enroll({ controlPlaneUrl: "http://127.0.0.1:1", bootstrapToken: "sbt", fetch: fakeControlPlane(both) });
+ const agentSession = await agent.join({ refreshLeadMs: 0, routerAddresses: [routerBAddr] });
+ const caller = await AgentMesh.enroll({ controlPlaneUrl: "http://127.0.0.1:1", bootstrapToken: "sbt", fetch: fakeControlPlane(both) });
+ const callerSession = await caller.join({ refreshLeadMs: 0, routerAddresses: [routerAddr], reserveRelay: false });
+ try {
+ assert.deepEqual(agentSession.routers.map((r) => r.peerId), [routerB.peerId.toString()]);
+ assert.deepEqual(callerSession.routers.map((r) => r.peerId), [router.peerId.toString()]);
+ assert.deepEqual(handshakesB, [agent.peerId]);
+
+ const verified = await callerSession.authenticate(agent.peerId, AbortSignal.timeout(15_000));
+ assert.equal(verified.peerId, agent.peerId);
+ // Router B admitted the caller on the way and stays admitted.
+ assert.deepEqual(handshakesB, [agent.peerId, caller.peerId]);
+ assert.deepEqual(callerSession.routers.map((r) => r.peerId).sort(), [router.peerId.toString(), routerB.peerId.toString()].sort());
+ } finally {
+ await callerSession.close();
+ await agentSession.close();
+ }
+ } finally {
+ await routerB.stop();
+ }
+});
+
test("join fails closed when the router is not a router", async () => {
// A relay whose credential lacks the router role must not admit us to the mesh.
const impostor = await createLibp2p({
diff --git a/sdk/js/src/session.ts b/sdk/js/src/session.ts
index 18c994b1..8973bd14 100644
--- a/sdk/js/src/session.ts
+++ b/sdk/js/src/session.ts
@@ -42,6 +42,13 @@ import { ingressHandler } from "./platform/ingress.ts";
import { BanSet, GOSSIP_EVENTS_TOPIC, MeshEvent_Type, verifyMeshEvent } from "./sync.ts";
export interface JoinOptions extends MeshHostOptions {
+ /**
+ * The routers to join through, `/…/p2p/<router>` multiaddrs, instead of
+ * the ones the credential lists. A peer behind a router not named here is
+ * still reached: connect() finds it through the routers' DHT and
+ * authenticates with the router that relays for it.
+ */
+ routerAddresses?: string[];
/**
* Reserve a relay slot on the first router that admits us, so peers can
* reach this member through the router. On by default; a member that
@@ -112,6 +119,8 @@ export interface ToolCallResult {
}
const DISCOVERY_TIMEOUT_MS = 5_000;
+/** Bounds the DHT walk connect() falls back to for a peer no admitted router relays. */
+const PEER_ROUTING_TIMEOUT_MS = 10_000;
const DEFAULT_REFRESH_LEAD_MS = 60 * 60 * 1000;
const DEFAULT_REFRESH_RETRY_MS = 30 * 1000;
@@ -213,10 +222,9 @@ export class MeshSession {
* reservation then still passes the router's check.
*/
#keepRouterAdmissions(): void {
- const routers = new Set(this.routers.map((r) => r.peerId));
this.node.addEventListener("connection:open", (evt) => {
const conn = evt.detail;
- if (this.#closed || !routers.has(conn.remotePeer.toString())) {
+ if (this.#closed || !this.routers.some((r) => r.peerId === conn.remotePeer.toString())) {
return;
}
void authenticateWithPeer(conn, this.mesh.authFrame(), this.mesh.credential.controlPlaneKeys).catch(() => {});
@@ -260,14 +268,118 @@ export class MeshSession {
/**
* Connects to a peer; see Peer for how it is named. Returns the
* connection, reused if one is already open. A banned peer is refused
- * here and by the connection gater.
+ * here and by the connection gater. A peer named by ID that no admitted
+ * router relays for is looked up in the routers' DHT, and failing that
+ * tried through every router the control plane lists that this member
+ * has not joined through; a relay opens a circuit only for a source it
+ * authenticated, so each such router is admitted first. Which router each
+ * side joined through does not decide whether they can talk.
*/
- connect(peer: Peer, signal?: AbortSignal): Promise<Connection> {
+ async connect(peer: Peer, signal?: AbortSignal): Promise<Connection> {
const { peerId, addrs } = this.dialTargets(peer);
if (peerId !== undefined && this.banned.has(peerId)) {
- return Promise.reject(new Error(`peer ${peerId} is banned by the control plane`));
+ throw new Error(`peer ${peerId} is banned by the control plane`);
+ }
+ const options = signal !== undefined ? { signal } : {};
+ try {
+ return await this.node.dial(addrs, options);
+ } catch (err) {
+ if (peerId === undefined || (typeof peer === "string" && peer.startsWith("/")) || isMultiaddr(peer) || signal?.aborted === true) {
+ throw err;
+ }
+ for (const more of [() => this.#routedAddresses(peerId, signal), () => this.#unjoinedRouterAddresses(peerId, signal)]) {
+ const routed = await more();
+ if (routed.length === 0) {
+ continue;
+ }
+ try {
+ return await this.node.dial(routed, options);
+ } catch (routedErr) {
+ err = routedErr;
+ }
+ }
+ throw err;
+ }
+ }
+
+ /**
+ * The relayed paths to a peer through the routers the control plane lists
+ * that this member has not joined through, each admitted first. The list
+ * is the one the credential carries, refreshed by every control plane
+ * pull, so a router that came up after join is tried too.
+ */
+ async #unjoinedRouterAddresses(peerId: string, signal?: AbortSignal): Promise<Multiaddr[]> {
+ const out: Multiaddr[] = [];
+ for (const text of this.mesh.credential.routerAddresses) {
+ let addr: Multiaddr;
+ try {
+ addr = multiaddr(text);
+ } catch {
+ continue;
+ }
+ const router = targetPeerOf(addr);
+ if (router === undefined || router === peerId || this.banned.has(router) || this.routers.some((r) => r.peerId === router)) {
+ continue;
+ }
+ try {
+ out.push((await this.#admitRouter(addr, signal)).addr.encapsulate(`/p2p-circuit/p2p/${peerId}`));
+ } catch {
+ continue;
+ }
+ }
+ return out;
+ }
+
+ /**
+ * The addresses the routers' DHT knows for a peer, its relayed ones
+ * through routers this member has admitted by then: a relay opens a
+ * circuit only for a source it authenticated, so a router met this way is
+ * dialed and passed the handshake first, and joins the admitted set.
+ */
+ async #routedAddresses(peerId: string, signal?: AbortSignal): Promise<Multiaddr[]> {
+ const lookup = signal ?? AbortSignal.timeout(PEER_ROUTING_TIMEOUT_MS);
+ let found: { multiaddrs: Multiaddr[] };
+ try {
+ found = await this.node.peerRouting.findPeer(peerIdFromString(peerId), { signal: lookup });
+ } catch {
+ return [];
+ }
+ const out: Multiaddr[] = [];
+ for (const ma of found.multiaddrs) {
+ const text = ma.toString();
+ const circuit = text.indexOf("/p2p-circuit");
+ if (circuit === -1) {
+ out.push(targetPeerOf(ma) === undefined ? ma.encapsulate(`/p2p/${peerId}`) : ma);
+ continue;
+ }
+ const relayAddr = multiaddr(text.slice(0, circuit));
+ const relay = targetPeerOf(relayAddr);
+ if (relay === undefined || this.banned.has(relay)) {
+ continue;
+ }
+ if (!this.routers.some((r) => r.peerId === relay)) {
+ try {
+ await this.#admitRouter(relayAddr, signal);
+ } catch {
+ continue;
+ }
+ }
+ out.push(relayAddr.encapsulate(`/p2p-circuit/p2p/${peerId}`));
+ }
+ return out;
+ }
+
+ /** Dials a router, runs the handshake and, its role verified, adds it to the admitted set. */
+ async #admitRouter(addr: Multiaddr, signal?: AbortSignal): Promise<AdmittedRouter> {
+ const conn = await this.node.dial(addr, signal !== undefined ? { signal } : {});
+ const credential = await authenticateWithPeer(conn, this.mesh.authFrame(), this.mesh.credential.controlPlaneKeys);
+ requireRole(credential, ROLE_ROUTER);
+ const peerId = conn.remotePeer.toString();
+ const router = { peerId, addr: connectedAddress(conn, peerId), credential };
+ if (!this.routers.some((r) => r.peerId === peerId)) {
+ this.routers.push(router);
}
- return this.node.dial(addrs, signal !== undefined ? { signal } : {});
+ return router;
}
/** The addresses connect() dials for a peer, in the order libp2p tries them. */
@@ -599,9 +711,9 @@ export class MeshSession {
/** Implements AgentMesh.join(); lives here to keep mesh.ts free of libp2p. */
export async function joinMesh(mesh: AgentMesh, options: JoinOptions = {}): Promise<MeshSession> {
- const routerAddrs = mesh.credential.routerAddresses.map((a) => multiaddr(a));
+ const routerAddrs = (options.routerAddresses ?? mesh.credential.routerAddresses).map((a) => multiaddr(a));
if (routerAddrs.length === 0) {
- throw new Error("credential lists no router addresses; the control plane had no active router at enrollment");
+ throw new Error(options.routerAddresses === undefined ? "credential lists no router addresses; the control plane had no active router at enrollment" : "routerAddresses names no router");
}
const banned = new BanSet();
diff --git a/sdk/python/src/agent_mesh/conformance_join.py b/sdk/python/src/agent_mesh/conformance_join.py
index d206a923..d97604dc 100644
--- a/sdk/python/src/agent_mesh/conformance_join.py
+++ b/sdk/python/src/agent_mesh/conformance_join.py
@@ -104,6 +104,8 @@ async def _handle(session: MeshSession, command: dict) -> dict:
return {"cmd": cmd, "ok": True, "is_error": result.is_error, "text": result.text}
if cmd == "peers":
return {"cmd": cmd, "authenticated_peers": sorted(session.authenticated_peers)}
+ if cmd == "routers":
+ return {"cmd": cmd, "ok": True, "routers": [r.peer_id for r in session.routers], "relay_addresses": session.relay_addresses}
if cmd == "sync":
result = await session.sync()
return {
@@ -171,7 +173,11 @@ async def main() -> None:
poll_interval=0.2,
)
# The test drives every pull itself; only gossip events bring one forward.
- async with mesh.join(listen_addrs=listen, control_plane_sync_interval=0, control_plane_sync_jitter=0) as session:
+ # SAM_SDK_ROUTERS, peer IDs, joins through those routers only, so the test
+ # can put two members on different routers.
+ only = {p for p in os.environ.get("SAM_SDK_ROUTERS", "").split(",") if p}
+ router_addresses = [a for a in mesh.credential.router_addresses if any(a.endswith(f"/p2p/{p}") for p in only)] if only else None
+ async with mesh.join(listen_addrs=listen, router_addresses=router_addresses, control_plane_sync_interval=0, control_plane_sync_jitter=0) as session:
_emit(
{
"sdk": "python",
diff --git a/sdk/python/src/agent_mesh/discovery.py b/sdk/python/src/agent_mesh/discovery.py
index 238cb928..e887ae73 100644
--- a/sdk/python/src/agent_mesh/discovery.py
+++ b/sdk/python/src/agent_mesh/discovery.py
@@ -14,9 +14,10 @@
"""Service discovery as sam-node does it (internal/node/service.go): a provider
record in the mesh DHT under a key derived from the service. The lookup is a
-bounded Kademlia GET_PROVIDERS walk spoken directly on go-libp2p-kad-dht's
-protocol: py-libp2p 0.7's DHT client hardcodes the /ipfs prefix and the mesh
-uses /sam. An SDK member only looks records up; it announces none."""
+bounded Kademlia GET_PROVIDERS walk, and a peer lookup a FIND_NODE one,
+spoken directly on go-libp2p-kad-dht's protocol: py-libp2p's DHT client
+(0.8 included) hardcodes the /ipfs prefix and the mesh uses /sam. An SDK
+member only looks records up; it announces none."""
from __future__ import annotations
@@ -105,6 +106,59 @@ def _peer_infos(peers: Iterable[kad.Message.Peer]) -> list[PeerInfo]:
return out
+async def _query(host: IHost, peer_id: ID, req: kad.Message) -> kad.Message | None:
+ """One request to a peer's DHT, answered or None: a peer that does not
+ serve the DHT, or does not answer in time, is skipped."""
+ try:
+ stream = await open_stream(host, peer_id, DHT_PROTOCOL, _QUERY_TIMEOUT)
+ except Exception as err: # noqa: BLE001 - a peer that does not serve the DHT is skipped
+ logger.debug("dht: %s does not answer %s: %s", peer_id, DHT_PROTOCOL, err)
+ return None
+ try:
+ with trio.fail_after(_QUERY_TIMEOUT):
+ await stream.write(encode_varint_prefixed(req.SerializeToString()))
+ resp = kad.Message.FromString(await read_varint_prefixed_bytes(stream))
+ except Exception as err: # noqa: BLE001
+ logger.debug("dht: query to %s failed: %s", peer_id, err)
+ return None
+ finally:
+ await stream.close()
+ return resp if resp.type == req.type else None
+
+
+async def find_peer(host: IHost, target: ID, seeds: Iterable[ID]) -> list[multiaddr.Multiaddr]:
+ """The addresses the DHT knows for a peer, as go-libp2p-kad-dht's FindPeer
+ learns them: a router answers FIND_NODE for a peer it is connected to with
+ that peer and its addresses, its relayed one included, and names the
+ routers closer to it otherwise. Empty when nobody knows the peer."""
+ asked: set[ID] = set()
+ frontier: list[ID] = list(seeds)
+ self_id = host.get_id()
+ req = kad.Message(type=kad.Message.FIND_NODE, key=target.to_bytes())
+ for _ in range(_MAX_ROUNDS):
+ batch = [p for p in frontier if p not in asked and p != self_id][:_MAX_PEERS_PER_ROUND]
+ if not batch:
+ break
+ frontier = []
+ for peer_id in batch:
+ asked.add(peer_id)
+ resp = await _query(host, peer_id, req)
+ if resp is None:
+ continue
+ for info in _peer_infos(resp.closerPeers):
+ if info.peer_id == target and info.addrs:
+ return list(info.addrs)
+ if info.peer_id in asked or info.peer_id in (self_id, target):
+ continue
+ if info.peer_id not in host.get_connected_peers():
+ try:
+ await host.connect(info)
+ except Exception: # noqa: BLE001 - unreachable closer peers are skipped
+ continue
+ frontier.append(info.peer_id)
+ return []
+
+
async def find_providers(host: IHost, key: bytes, seeds: Iterable[ID], limit: int = 20) -> list[DiscoveredProvider]:
"""Asks the seed peers (the routers we are connected to) for providers of
key and follows the closer peers they name for a few rounds."""
diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.py
index ab983bc6..2aa52c90 100644
--- a/sdk/python/src/agent_mesh/session.py
+++ b/sdk/python/src/agent_mesh/session.py
@@ -40,7 +40,7 @@ from ._proto import sam_pb2 as pb
from .auth import AUTH_PROTOCOL, auth_stream_handler, authenticate_with_peer
from .authorizer import ProviderAuthorizerOptions
from .biscuit import ROLE_ROUTER, VerifiedBiscuit, require_role
-from .discovery import DiscoveredProvider, find_providers, parse_service_target, service_key
+from .discovery import DiscoveredProvider, find_peer, find_providers, parse_service_target, service_key
from .host import create_mesh_host, dial_addrs, peer_info
from .httpx_transport import MESH_PATH_PREFIX
from .identity import canonical_peer_id
@@ -204,7 +204,12 @@ class MeshSession:
async def connect(self, peer: Peer) -> ID:
"""Connects to a peer, see `Peer`, and returns its peer ID. A banned
- peer is refused."""
+ peer is refused. A peer named by ID that no admitted router relays for
+ is looked up in the routers' DHT, and failing that tried through every
+ router the control plane lists that this member has not joined
+ through; a relay opens a circuit only for a source it authenticated,
+ so each such router is admitted first. Which router each side joined
+ through does not decide whether they can talk."""
if isinstance(peer, multiaddr.Multiaddr) or (isinstance(peer, str) and peer.startswith("/")):
return await self._connect_addr(multiaddr.Multiaddr(str(peer)))
if isinstance(peer, str):
@@ -236,8 +241,76 @@ class MeshSession:
return target
except Exception as err: # noqa: BLE001 - the next router is tried
failures.append(f"via router {r.peer_id}: {err}")
+ for more in (self._routed_addresses, self._unjoined_router_addresses):
+ for ma in await more(target):
+ try:
+ await self._connect_addr(ma)
+ return target
+ except Exception as err: # noqa: BLE001 - the next address is tried
+ failures.append(f"{ma}: {err}")
raise ConnectionError(f"cannot reach {target}:\n " + "\n ".join(failures))
+ async def _unjoined_router_addresses(self, target: ID) -> list[multiaddr.Multiaddr]:
+ """The relayed paths to a peer through the routers the control plane
+ lists that this member has not joined through, each admitted first.
+ The list is the one the credential carries, refreshed by every
+ control plane pull, so a router that came up after join is tried too."""
+ out: list[multiaddr.Multiaddr] = []
+ for text in self.mesh.credential.router_addresses:
+ try:
+ addr = multiaddr.Multiaddr(text)
+ router = info_from_p2p_addr(addr).peer_id
+ except Exception: # noqa: BLE001 - not a router address
+ continue
+ if router == target or str(router) in self.banned or any(r.peer_id == str(router) for r in self.routers):
+ continue
+ try:
+ admitted = await self._admit_router(addr)
+ except Exception as err: # noqa: BLE001 - a router that does not admit us is not a path
+ logger.debug("router %s did not admit us: %s", router, err)
+ continue
+ out.append(multiaddr.Multiaddr(f"{admitted.addr}/p2p-circuit/p2p/{target}"))
+ return out
+
+ async def _routed_addresses(self, target: ID) -> list[multiaddr.Multiaddr]:
+ """The addresses the routers' DHT knows for a peer, relayed ones
+ through routers this member has admitted by then: a relay opens a
+ circuit only for a source it authenticated, so a router met this way
+ is dialed and passed the handshake first, and joins the admitted set."""
+ out: list[multiaddr.Multiaddr] = []
+ seeds = [ID.from_base58(r.peer_id) for r in self.routers]
+ for ma in await find_peer(self.host, target, seeds):
+ text = str(ma)
+ if "/p2p-circuit" not in text:
+ out.append(ma if text.endswith(f"/p2p/{target}") else multiaddr.Multiaddr(f"{text}/p2p/{target}"))
+ continue
+ relay_addr = multiaddr.Multiaddr(text[: text.index("/p2p-circuit")])
+ try:
+ relay = info_from_p2p_addr(relay_addr).peer_id
+ except Exception: # noqa: BLE001 - a circuit address naming no relay is useless
+ continue
+ if str(relay) in self.banned:
+ continue
+ if not any(r.peer_id == str(relay) for r in self.routers):
+ try:
+ await self._admit_router(relay_addr)
+ except Exception as err: # noqa: BLE001 - a relay that is not a router of this mesh is not used
+ logger.debug("router %s named by the DHT did not admit us: %s", relay, err)
+ continue
+ out.append(multiaddr.Multiaddr(f"{relay_addr}/p2p-circuit/p2p/{target}"))
+ return out
+
+ async def _admit_router(self, addr: multiaddr.Multiaddr) -> AdmittedRouter:
+ """Dials a router, runs the handshake and, its role verified, adds it
+ to the admitted set."""
+ info = await peer_info(addr)
+ await dial(self.host, info)
+ credential = await _authenticate_router(self.host, self.mesh, info.peer_id)
+ router = AdmittedRouter(peer_id=str(info.peer_id), addr=addr, credential=credential)
+ if not any(r.peer_id == router.peer_id for r in self.routers):
+ self.routers.append(router)
+ return router
+
async def _connect_addr(self, ma: multiaddr.Multiaddr) -> ID:
"""Dials one multiaddr, through a relay when it says `/p2p-circuit`."""
if "/p2p-circuit" in str(ma):
@@ -516,6 +589,7 @@ async def join_mesh(
mesh: "AgentMesh",
*,
listen_addrs: Sequence[str] = (),
+ router_addresses: Optional[Sequence[str]] = None,
reserve: bool = True,
refresh_lead: float = DEFAULT_REFRESH_LEAD,
refresh_retry: float = DEFAULT_REFRESH_RETRY,
@@ -525,10 +599,13 @@ async def join_mesh(
control_plane_sync_interval: float = DEFAULT_CONTROL_PLANE_SYNC,
control_plane_sync_jitter: float = DEFAULT_CONTROL_PLANE_SYNC_JITTER,
) -> AsyncIterator[MeshSession]:
- """Implements AgentMesh.join(); lives here to keep mesh.py free of libp2p."""
- router_addrs = [multiaddr.Multiaddr(a) for a in mesh.credential.router_addresses]
+ """Implements AgentMesh.join(); lives here to keep mesh.py free of libp2p.
+ router_addresses names the routers to join through instead of the ones the
+ credential lists; a peer behind another router is still reached, see
+ MeshSession.connect."""
+ router_addrs = [multiaddr.Multiaddr(a) for a in (mesh.credential.router_addresses if router_addresses is None else router_addresses)]
if not router_addrs:
- raise RuntimeError("credential lists no router addresses; the control plane had no active router at enrollment")
+ raise RuntimeError("credential lists no router addresses; the control plane had no active router at enrollment" if router_addresses is None else "router_addresses names no router")
host, listen = create_mesh_host(mesh.identity, listen_addrs)
authenticated: dict[str, datetime] = {}
diff --git a/tests/integration/sdk_mesh_test.go b/tests/integration/sdk_mesh_test.go
index b813b829..e73fb31f 100644
--- a/tests/integration/sdk_mesh_test.go
+++ b/tests/integration/sdk_mesh_test.go
@@ -715,20 +715,149 @@ func TestNativeSDKsMesh(t *testing.T) {
}
}
+// TestNativeSDKsAcrossRouters is the datapath between two members that joined
+// through different routers: one member per SDK on router A and one on router
+// B, none with an address of its own, so the relayed path is the only one.
+// Every member on one router calls the agent of every member on the other by
+// peer ID, over /libp2p-http, and authenticates with it. Which router a peer
+// happened to reserve on must not decide whether it can be reached; the
+// caller finds the peer through the routers and is admitted by the one that
+// relays for it on the way.
+func TestNativeSDKsAcrossRouters(t *testing.T) {
+ mesh := startSDKMesh(t)
+ if len(mesh.routerAddrs) < 2 {
+ t.Fatalf("the fixture runs %d router(s), want two", len(mesh.routerAddrs))
+ }
+
+ type placed struct {
+ *sdkMember
+ router string
+ }
+ var members []placed
+ for _, launcher := range sdkMemberLaunchers {
+ for _, routerAddr := range mesh.routerAddrs {
+ cmd, skip := launcher.cmd(mesh.root)
+ if skip != "" {
+ t.Logf("%s SDK skipped: %s", launcher.name, skip)
+ break
+ }
+ router := extractPeerID(routerAddr)
+ m := launchSDKMember(t, launcher.name+"@"+router[len(router)-6:], cmd, mesh.root, mesh.baseURL, mesh.adminToken, "SAM_SDK_ROUTERS="+router)
+ if got := routerIDs(m.report); !reflect.DeepEqual(got, []string{router}) {
+ t.Fatalf("%s joined through %v, want %s only", m.name, got, router)
+ }
+ for _, a := range m.report.RelayAddresses {
+ if !strings.HasPrefix(a, routerAddr) {
+ t.Fatalf("%s reserved on %s, want %s", m.name, a, routerAddr)
+ }
+ }
+ if len(m.report.DirectAddresses) != 0 {
+ t.Fatalf("%s listens on %v, want the relayed path to be the only one", m.name, m.report.DirectAddresses)
+ }
+ m.accept(t, "agent")
+ members = append(members, placed{m, router})
+ }
+ }
+ if len(members) == 0 {
+ t.Skip("no SDK toolchain available; see sdk/README.md")
+ }
+ for _, m := range members {
+ waitForPeerOnRouter(t, mesh.cpPort, mesh.adminToken, m.report.PeerID, 5*time.Second)
+ }
+
+ // A pair that talked once stays connected, so only the first contact
+ // between two members has to find its way through the other router.
+ connected := map[[2]string]bool{}
+ for _, caller := range members {
+ for _, agent := range members {
+ if caller.router == agent.router {
+ continue
+ }
+ t.Run(caller.name+"-reaches-"+agent.name, func(t *testing.T) {
+ pair := [2]string{caller.report.PeerID, agent.report.PeerID}
+ if pair[0] > pair[1] {
+ pair[0], pair[1] = pair[1], pair[0]
+ }
+ firstContact := !connected[pair]
+ connected[pair] = true
+
+ // The agent names the verified caller in its answer.
+ res := caller.http(t, agent.report.PeerID, "a2a://agent", "/hello")
+ if res.Status != 200 || !strings.Contains(res.Body, caller.report.PeerID) {
+ t.Fatalf("%s -> %s: %d %s", caller.name, agent.name, res.Status, res.Body)
+ }
+ verified := caller.auth(t, agent.report.PeerID)
+ if verified.PeerID != agent.report.PeerID {
+ t.Fatalf("%s authenticated %s, want %s", caller.name, verified.PeerID, agent.report.PeerID)
+ }
+ // The agent's router admitted the caller on the way and stays admitted.
+ if got := caller.routers(t); firstContact && !contains(got, agent.router) {
+ t.Fatalf("%s admitted routers %v after reaching a peer on %s", caller.name, got, agent.router)
+ }
+ })
+ }
+ }
+ for _, m := range members {
+ m.quit(t)
+ }
+}
+
+// routerIDs lists the routers a join report says admitted the member.
+func routerIDs(report sdkJoinReport) []string {
+ var ids []string
+ for _, r := range report.Routers {
+ ids = append(ids, r.PeerID)
+ }
+ return ids
+}
+
func startSDKMember(t *testing.T, name string, cmd *exec.Cmd, root, baseURL, adminToken string, routerAddrs []string) *sdkMember {
t.Helper()
routerAddr, routerPeer := routerAddrs[0], extractPeerID(routerAddrs[0])
+ m := launchSDKMember(t, name, cmd, root, baseURL, adminToken, "SAM_SDK_LISTEN_ADDRS=/ip4/127.0.0.1/tcp/0")
+ // Every router the control plane named admitted the member, in the
+ // control plane's order: a peer reserves on the first, and a caller that
+ // knew only its own first router could not reach a peer whose list
+ // started elsewhere.
+ var admitted []string
+ for _, r := range m.report.Routers {
+ admitted = append(admitted, r.PeerID)
+ if !contains(r.Roles, api.RoleRouter) {
+ t.Fatalf("%s: router %s credential roles %v lack %s", name, r.PeerID, r.Roles, api.RoleRouter)
+ }
+ }
+ var want []string
+ for _, a := range routerAddrs {
+ want = append(want, extractPeerID(a))
+ }
+ if !reflect.DeepEqual(admitted, want) {
+ t.Fatalf("%s report routers %v, want every router the control plane listed %v", name, admitted, want)
+ }
+ if len(m.report.RelayAddresses) == 0 {
+ t.Fatalf("%s: no relay address, the router did not grant a reservation", name)
+ }
+ for _, a := range m.report.RelayAddresses {
+ if !strings.HasPrefix(a, routerAddr) || !strings.HasSuffix(a, "/p2p-circuit/p2p/"+m.report.PeerID) {
+ t.Fatalf("%s relay address %s is not <router>/p2p-circuit/p2p/<self> for the first router %s (%s)", name, a, routerPeer, routerAddr)
+ }
+ }
+ return m
+}
+
+// launchSDKMember starts a conformance-join runner with a fresh bootstrap
+// token and the labels the matrix uses, plus env, and reads its join report.
+func launchSDKMember(t *testing.T, name string, cmd *exec.Cmd, root, baseURL, adminToken string, env ...string) *sdkMember {
+ t.Helper()
tokenPath := filepath.Join(t.TempDir(), "bootstrap.token")
if err := os.WriteFile(tokenPath, []byte(mintBootstrapToken(t, baseURL, adminToken)+"\n"), 0o600); err != nil {
t.Fatalf("failed to write bootstrap token: %v", err)
}
- cmd.Env = append(os.Environ(),
+ cmd.Env = append(append(os.Environ(),
"SAM_CONTROL_PLANE_URL="+baseURL,
"SAM_BOOTSTRAP_TOKEN_PATH="+tokenPath,
"SAM_SDK_STATE_DIR="+filepath.Join(t.TempDir(), "state"),
- "SAM_SDK_LISTEN_ADDRS=/ip4/127.0.0.1/tcp/0",
"SAM_SDK_LABELS="+labelsEnv(sdkMeshLabels),
- )
+ ), env...)
cmd.Dir = root
m := &sdkMember{name: name, cmd: cmd, stderr: &bytes.Buffer{}}
cmd.Stderr = m.stderr
@@ -760,35 +889,9 @@ func startSDKMember(t *testing.T, name string, cmd *exec.Cmd, root, baseURL, adm
if line := m.readLine(t, 20*time.Second); json.Unmarshal(line, &m.report) != nil {
t.Fatalf("%s member printed no join report, got: %q", name, line)
}
- // Every router the control plane named admitted the member, in the
- // control plane's order: a peer reserves on the first, and a caller that
- // knew only its own first router could not reach a peer whose list
- // started elsewhere.
- var admitted []string
- for _, r := range m.report.Routers {
- admitted = append(admitted, r.PeerID)
- if !contains(r.Roles, api.RoleRouter) {
- t.Fatalf("%s: router %s credential roles %v lack %s", name, r.PeerID, r.Roles, api.RoleRouter)
- }
- }
- var want []string
- for _, a := range routerAddrs {
- want = append(want, extractPeerID(a))
- }
- if !reflect.DeepEqual(admitted, want) {
- t.Fatalf("%s report routers %v, want every router the control plane listed %v", name, admitted, want)
- }
if _, err := peer.Decode(m.report.PeerID); err != nil {
t.Fatalf("%s report peer_id %q: %v", name, m.report.PeerID, err)
}
- if len(m.report.RelayAddresses) == 0 {
- t.Fatalf("%s: no relay address, the router did not grant a reservation", name)
- }
- for _, a := range m.report.RelayAddresses {
- if !strings.HasPrefix(a, routerAddr) || !strings.HasSuffix(a, "/p2p-circuit/p2p/"+m.report.PeerID) {
- t.Fatalf("%s relay address %s is not <router>/p2p-circuit/p2p/<self> for the first router %s (%s)", name, a, routerPeer, routerAddr)
- }
- }
return m
}
@@ -1051,6 +1154,23 @@ func (m *sdkMember) http(t *testing.T, addr, service, path string) sdkHTTPResult
return res
}
+// routers asks the member which routers admitted it so far.
+func (m *sdkMember) routers(t *testing.T) []string {
+ t.Helper()
+ var res struct {
+ OK bool `json:"ok"`
+ Error string `json:"error"`
+ Routers []string `json:"routers"`
+ }
+ if line := m.send(t, map[string]string{"cmd": "routers"}); json.Unmarshal(line, &res) != nil {
+ t.Fatalf("%s member: routers answered %q", m.name, line)
+ }
+ if !res.OK {
+ t.Fatalf("%s member: routers: %s", m.name, res.Error)
+ }
+ return res.Routers
+}
+
func (m *sdkMember) quit(t *testing.T) {
t.Helper()
m.send(t, map[string]string{"cmd": "quit"})