Skip to content

Agent Mesh

sdk/python: follow the mesh's replication of closer routers in a walk

Antonio Ojeac4435222cb14
diff --git a/sdk/python/src/agent_mesh/discovery.py b/sdk/python/src/agent_mesh/discovery.py
index f4571746..3d465189 100644
--- a/sdk/python/src/agent_mesh/discovery.py
+++ b/sdk/python/src/agent_mesh/discovery.py
@@ -50,6 +50,13 @@ _MAX_ROUNDS = 3
_MAX_PEERS_PER_ROUND = 8
MAX_DHT_MESSAGE_BYTES = 256 * 1024
+# How many routers hold each record and answer each lookup when the control
+# plane publishes no directory parameters on /info: api.DefaultDHTReplication.
+# A member reads the mesh's value from /info (Mesh.dht_replication) and
+# follows that many of the closer routers an answer names, as agentmesh-node
+# asks that many at a time.
+DEFAULT_DHT_REPLICATION = 3
+
@dataclass
class DiscoveredProvider:
@@ -109,7 +116,7 @@ async def _query(host: IHost, peer_id: ID, req: kad.Message) -> kad.Message | No
return resp if resp.type == req.type else None
-async def find_peer(host: IHost, target: ID, seeds: Iterable[ID]) -> list[multiaddr.Multiaddr]:
+async def find_peer(host: IHost, target: ID, seeds: Iterable[ID], replication: int = DEFAULT_DHT_REPLICATION) -> list[multiaddr.Multiaddr]:
"""The addresses the DHT knows for a peer, as go-libp2p-kad-dht's FindPeer
learns them: a router answers FIND_NODE for a peer it is connected to with
that peer and its addresses, its relayed one included, and names the
@@ -123,11 +130,11 @@ async def find_peer(host: IHost, target: ID, seeds: Iterable[ID]) -> list[multia
return True
return False
- await _walk(host, kad.Message(type=kad.Message.FIND_NODE, key=target.to_bytes()), seeds, take, skip={target})
+ await _walk(host, kad.Message(type=kad.Message.FIND_NODE, key=target.to_bytes()), seeds, take, skip={target}, replication=replication)
return found
-async def find_providers(host: IHost, key: bytes, seeds: Iterable[ID], limit: int = 20) -> list[DiscoveredProvider]:
+async def find_providers(host: IHost, key: bytes, seeds: Iterable[ID], limit: int = 20, replication: int = DEFAULT_DHT_REPLICATION) -> list[DiscoveredProvider]:
"""Asks the seed peers (the routers we are connected to) for providers of
key and follows the closer peers they name for a few rounds."""
found: dict[str, DiscoveredProvider] = {}
@@ -143,21 +150,31 @@ async def find_providers(host: IHost, key: bytes, seeds: Iterable[ID], limit: in
entry.addrs.append(str(ma))
return bool(found)
- await _walk(host, kad.Message(type=kad.Message.GET_PROVIDERS, key=key), seeds, take)
+ await _walk(host, kad.Message(type=kad.Message.GET_PROVIDERS, key=key), seeds, take, replication=replication)
return list(found.values())[:limit]
-async def _walk(host: IHost, req: kad.Message, seeds: Iterable[ID], take: Callable[[kad.Message], bool], skip: frozenset[ID] | set[ID] = frozenset()) -> None:
+async def _walk(
+ host: IHost,
+ req: kad.Message,
+ seeds: Iterable[ID],
+ take: Callable[[kad.Message], bool],
+ skip: frozenset[ID] | set[ID] = frozenset(),
+ replication: int = DEFAULT_DHT_REPLICATION,
+) -> None:
"""The Kademlia walk both lookups share. A round asks its peers at once
and ends when an answer satisfies `take`; only otherwise are the closer
peers the answers named dialed, at once as well. A round thus costs one
query and one dial timeout, not one per peer: right after a rollout the
routers' tables still name the pods it replaced, and the control plane
may list a router this member cannot reach. A peer that did not answer
- its dial is not dialed again."""
+ its dial is not dialed again. A router names the routers closest to the
+ key in order, and a record lives on the `replication` closest, so only
+ that many of them are followed from one round to the next."""
self_id = host.get_id()
asked: set[ID] = set()
frontier: list[ID] = list(seeds)
+ fanout = min(max(replication, 1), _MAX_PEERS_PER_ROUND)
for _ in range(_MAX_ROUNDS):
batch = [p for p in frontier if p not in asked and p != self_id][:_MAX_PEERS_PER_ROUND]
if not batch:
@@ -183,6 +200,8 @@ async def _walk(host: IHost, req: kad.Message, seeds: Iterable[ID], take: Callab
closer: dict[ID, PeerInfo] = {}
for resp in answers:
for info in _peer_infos(resp.closerPeers):
+ if len(closer) >= fanout:
+ break
if info.peer_id not in asked and info.peer_id != self_id and info.peer_id not in skip:
closer.setdefault(info.peer_id, info)
connected = set(host.get_connected_peers())
diff --git a/sdk/python/src/agent_mesh/mesh.py b/sdk/python/src/agent_mesh/mesh.py
index dc6c2ba1..a7132837 100644
--- a/sdk/python/src/agent_mesh/mesh.py
+++ b/sdk/python/src/agent_mesh/mesh.py
@@ -24,6 +24,7 @@ from typing import Callable, Mapping, Optional
from ._proto import agentmesh_pb2 as pb
from .controlplane import ROLE_NODE, ControlPlaneClient, Enrollment, KeysNotTrustedError, Transport
from .credential import MeshCredential, encode_auth_frame
+from .discovery import DEFAULT_DHT_REPLICATION
from .identity import Identity
_IDENTITY_FILE = "identity.key"
@@ -98,6 +99,7 @@ class AgentMesh:
self.control_plane = control_plane
self._credential = credential
self._routers: list[pb.RouterInfo] = []
+ self._dht_replication = DEFAULT_DHT_REPLICATION
self._state_dir = state_dir
self._jwt_source = jwt_source
# refresh() and sync_control_plane() run in worker threads of the
@@ -118,6 +120,14 @@ class AgentMesh:
"""The router catalog (ControlPlaneInfoResponse.routers) from the most recent /info pull."""
return list(self._routers)
+ @property
+ def dht_replication(self) -> int:
+ """How many routers hold each record and answer each lookup on this mesh,
+ as /info last published it (ControlPlaneInfoResponse.directory);
+ DEFAULT_DHT_REPLICATION until the control plane has answered or when it
+ publishes none."""
+ return self._dht_replication
+
def attenuate(self, rule: pb.TaskAuthorizationRule) -> MeshCredential:
"""Returns a new MeshCredential with a tar_block appended offline in memory."""
return self._credential.attenuate(rule)
@@ -316,6 +326,8 @@ class AgentMesh:
info = self.control_plane.info()
routers = list(info.routers)
self._routers = routers
+ if info.HasField("directory") and info.directory.replication >= 1:
+ self._dht_replication = info.directory.replication
if info.router_addresses:
self._credential = replace(self._credential, router_addresses=list(info.router_addresses))
elif routers:
diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.py
index f1add048..08f3734d 100644
--- a/sdk/python/src/agent_mesh/session.py
+++ b/sdk/python/src/agent_mesh/session.py
@@ -585,7 +585,7 @@ class MeshSession:
relays: list[multiaddr.Multiaddr] = []
seeds = [ID.from_base58(r.peer_id) for r in self.routers]
suffix = f"/p2p/{target}"
- for ma in await find_peer(self.host, target, seeds):
+ for ma in await find_peer(self.host, target, seeds, self.mesh.dht_replication):
text = str(ma)
if "/p2p-circuit" not in text:
try:
@@ -649,7 +649,7 @@ class MeshSession:
if service_type not in ("mcp", "inference", "a2a", "egress"):
raise ValueError(f"service type must be mcp, inference, a2a or egress, got {service_type!r}")
seeds = [ID.from_base58(r.peer_id) for r in self.routers]
- return await find_providers(self.host, service_key(service_type, service_name), seeds, limit)
+ return await find_providers(self.host, service_key(service_type, service_name), seeds, limit, self.mesh.dht_replication)
def open_mcp(
self,
diff --git a/sdk/python/tests/test_discovery.py b/sdk/python/tests/test_discovery.py
index 3da18716..dd849719 100644
--- a/sdk/python/tests/test_discovery.py
+++ b/sdk/python/tests/test_discovery.py
@@ -254,3 +254,29 @@ def test_read_bounded_varint_prefixed_bytes_rejects_oversized_prefix_before_read
trio.run(main)
+
+
+def test_the_walk_follows_only_the_replication_closest_routers():
+ """A router names the routers closest to the key in order, as many as it
+ knows; a record lives on the mesh's replication of them, so only that
+ many are followed. With the default of three the fourth is never dialed;
+ with a replication of one, only the first."""
+
+ def knows_nothing(req):
+ return kad.Message(
+ type=req.type,
+ closerPeers=[
+ _peer(GONE_POD, "/ip4/10.84.1.7/tcp/5002"),
+ _peer(GONE_POD_2, "/ip4/10.84.2.9/tcp/5002"),
+ _peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501"),
+ _peer(TARGET, "/ip4/10.0.0.9/tcp/4001"),
+ ],
+ )
+
+ async def main(replication, want):
+ host = _Host({ROUTER: knows_nothing, ROUTER_B: knows_nothing})
+ assert await find_providers(host, b"key", [ROUTER, ROUTER_B], replication=replication) == []
+ assert sorted(map(str, host.dials)) == sorted(map(str, want))
+
+ trio.run(main, 3, [GONE_POD, GONE_POD_2, DARK_ROUTER], clock=trio.testing.MockClock(autojump_threshold=0))
+ trio.run(main, 1, [GONE_POD], clock=trio.testing.MockClock(autojump_threshold=0))