Skip to content

Agent Mesh

sdk/python: ask the held routers for the directory, floodsub for events

Antonio Ojea40f1b0da5e3b
diff --git a/sdk/python/src/agent_mesh/__init__.py b/sdk/python/src/agent_mesh/__init__.py
index 51d71125..6b85a6b9 100644
--- a/sdk/python/src/agent_mesh/__init__.py
+++ b/sdk/python/src/agent_mesh/__init__.py
@@ -54,7 +54,7 @@ from .controlplane import (
verify_keys_response,
)
from .credential import MeshCredential, decode_auth_response, encode_auth_frame
-from .discovery import DHT_PROTOCOL, DiscoveredProvider, find_providers, parse_service_target, service_key
+from .discovery import DIRECTORY_PROTOCOL, DiscoveredProvider, DirectoryError, find_providers, parse_service_target, peer_key, service_key
from .httpx_transport import MESH_PATH_PREFIX, MeshTransport, split_mesh_url
from .identity import Identity, canonical_peer_id, libp2p_public_key, peer_id_from_public_key, verify_ed25519
from .libp2p_http import (
@@ -125,7 +125,8 @@ __all__ = [
"CredentialRetiredError",
"DEFAULT_A2A_NAME",
"DEFAULT_ROUTERS",
- "DHT_PROTOCOL",
+ "DIRECTORY_PROTOCOL",
+ "DirectoryError",
"DiscoveredProvider",
"Enrollment",
"EnrollmentRejectedError",
@@ -207,6 +208,7 @@ __all__ = [
"revocations_challenge",
"seal_biscuit",
"select_routers",
+ "peer_key",
"service_key",
"rewrite_agent_card",
"split_mesh_url",
diff --git a/sdk/python/src/agent_mesh/discovery.py b/sdk/python/src/agent_mesh/discovery.py
index 272fa4ec..06e9bb56 100644
--- a/sdk/python/src/agent_mesh/discovery.py
+++ b/sdk/python/src/agent_mesh/discovery.py
@@ -12,67 +12,73 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Service discovery as agentmesh-node does it (internal/node/service.go): a provider
-record in the mesh DHT under a key derived from the service. The lookup is a
-bounded Kademlia GET_PROVIDERS walk, and a peer lookup a FIND_NODE one,
-spoken directly on go-libp2p-kad-dht's protocol: py-libp2p's DHT client
-(0.8 included) hardcodes the /ipfs prefix and the mesh uses /mesh. An SDK
-member only looks records up; it announces none."""
+"""The mesh's directory as a member uses it, as agentmesh-node does
+(internal/node/directory_client.go): the member keeps no DHT and asks the
+routers it holds over /mesh/directory/1.0.0, one DirectoryRequest per
+stream, one DirectoryResponse back. A request goes to one held router and
+to the next when it fails. The routers run the lookup across their own DHT.
+A key is the sha256 multihash of "mesh:service:<type>[:<name>]" for a
+service and of "mesh:peer:<peer_id>" for a member."""
from __future__ import annotations
import hashlib
import logging
from dataclasses import dataclass, field
-from typing import Awaitable, Callable, Iterable, Literal
+from typing import Iterable, Literal
import multiaddr
import trio
from libp2p.abc import IHost
from libp2p.custom_types import TProtocol
-from libp2p.kad_dht.pb import kademlia_pb2 as kad
from libp2p.peer.id import ID
-from libp2p.peer.peerinfo import PeerInfo
from libp2p.utils.varint import encode_varint_prefixed
+from ._proto import agentmesh_pb2 as pb
from .auth import read_bounded_varint_prefixed_bytes
-from .host import dial, open_stream
+from .host import open_stream
logger = logging.getLogger("agent_mesh")
-# go-libp2p-kad-dht with dht.ProtocolPrefix("/mesh").
-DHT_PROTOCOL = TProtocol("/mesh/kad/1.0.0")
+# api.DirectoryProtocolID.
+DIRECTORY_PROTOCOL = TProtocol("/mesh/directory/1.0.0")
ServiceType = Literal["mcp", "inference", "a2a", "egress"]
-_QUERY_TIMEOUT = 5.0
-_MAX_ROUNDS = 3
-_MAX_PEERS_PER_ROUND = 8
-MAX_DHT_MESSAGE_BYTES = 256 * 1024
-
-# How many routers hold each record and answer each lookup when the control
-# plane publishes no directory parameters on /info: api.DefaultDHTReplication.
-# A member reads the mesh's value from /info (Mesh.dht_replication) and
-# follows that many of the closer routers an answer names, as agentmesh-node
-# asks that many at a time.
-DEFAULT_DHT_REPLICATION = 3
+_REQUEST_TIMEOUT = 10.0
+MAX_DIRECTORY_MESSAGE_BYTES = 64 * 1024
+DEFAULT_LIMIT = 20
@dataclass
class DiscoveredProvider:
- """A peer the DHT names as offering a service."""
+ """A peer the directory names as offering a service."""
peer_id: str
- # Addresses the provider advertised; may be empty when the record carried none.
+ # Addresses the provider is reached at, each ending in its peer ID; may
+ # be empty when the record carried none.
addrs: list[str] = field(default_factory=list)
+class DirectoryError(Exception):
+ """No held router answered a directory request."""
+
+
+def _key(name: str) -> bytes:
+ """The sha256 multihash of a name: the two-byte header and the digest."""
+ return b"\x12\x20" + hashlib.sha256(name.encode()).digest()
+
+
def service_key(service_type: ServiceType, name: str | None = None) -> bytes:
- """The DHT key of a service: the sha256 multihash of "mesh:service:<type>[:<name>]".
- go-libp2p-kad-dht keys provider records by the multihash, not the CID."""
+ """The directory key of a service type, or of one named service of that
+ type: "mesh:service:<type>" or "mesh:service:<type>:<name>" (api.ServiceKey)."""
parts = ["mesh:service", service_type] + ([name] if name else [])
- digest = hashlib.sha256(":".join(parts).encode()).digest()
- return b"\x12\x20" + digest
+ return _key(":".join(parts))
+
+
+def peer_key(peer_id: ID) -> bytes:
+ """The directory key a member announces where it is reached under (api.PeerKey)."""
+ return _key(f"mesh:peer:{peer_id}")
def parse_service_target(target: str) -> tuple[str, str]:
@@ -85,188 +91,72 @@ def parse_service_target(target: str) -> tuple[str, str]:
return scheme, name
-def _peer_infos(peers: Iterable[kad.Message.Peer]) -> list[PeerInfo]:
- out = []
- for p in peers:
- try:
- addrs = [multiaddr.Multiaddr(a) for a in p.addrs]
- out.append(PeerInfo(ID(p.id), addrs))
- except Exception: # noqa: BLE001 - a malformed entry from a peer is dropped, not fatal
- continue
- return out
-
-
-async def _query(host: IHost, peer_id: ID, req: kad.Message) -> kad.Message | None:
- """One request to a peer's DHT, answered or None: a peer that does not
- serve the DHT, or does not answer in time, is skipped."""
- try:
- stream = await open_stream(host, peer_id, DHT_PROTOCOL, _QUERY_TIMEOUT)
- except Exception as err: # noqa: BLE001 - a peer that does not serve the DHT is skipped
- logger.debug("dht: %s does not answer %s: %s", peer_id, DHT_PROTOCOL, err)
- return None
+async def _ask(host: IHost, router: ID, req: pb.DirectoryRequest) -> pb.DirectoryResponse:
+ """One request to one router, answered or raised."""
+ stream = await open_stream(host, router, DIRECTORY_PROTOCOL, _REQUEST_TIMEOUT)
try:
- with trio.fail_after(_QUERY_TIMEOUT):
+ with trio.fail_after(_REQUEST_TIMEOUT):
await stream.write(encode_varint_prefixed(req.SerializeToString()))
- resp = kad.Message.FromString(await read_bounded_varint_prefixed_bytes(stream, MAX_DHT_MESSAGE_BYTES))
- except Exception as err: # noqa: BLE001
- logger.debug("dht: query to %s failed: %s", peer_id, err)
- return None
+ resp = pb.DirectoryResponse.FromString(await read_bounded_varint_prefixed_bytes(stream, MAX_DIRECTORY_MESSAGE_BYTES))
finally:
await stream.close()
- return resp if resp.type == req.type else None
-
-
-Admit = Callable[[ID], Awaitable[None]]
-"""Authenticates this member with a router before it is queried: a router
-serves its DHT to a peer only once that peer passed the handshake."""
-
-
-async def find_peer(
- host: IHost,
- target: ID,
- seeds: Iterable[ID],
- replication: int = DEFAULT_DHT_REPLICATION,
- admit: Admit | None = None,
-) -> list[multiaddr.Multiaddr]:
- """The addresses the DHT knows for a peer, as go-libp2p-kad-dht's FindPeer
- learns them: a router answers FIND_NODE for a peer it is connected to with
- that peer and its addresses, its relayed one included, and names the
- routers closer to it otherwise. Empty when nobody knows the peer."""
- found: list[multiaddr.Multiaddr] = []
-
- def take(resp: kad.Message) -> bool:
- for info in _peer_infos(resp.closerPeers):
- if info.peer_id == target and info.addrs:
- found.extend(info.addrs)
- return True
- return False
-
- await _walk(host, kad.Message(type=kad.Message.FIND_NODE, key=target.to_bytes()), seeds, take, skip={target}, replication=replication, admit=admit)
- return found
-
-
-async def find_providers(
- host: IHost,
- key: bytes,
- seeds: Iterable[ID],
- limit: int = 20,
- replication: int = DEFAULT_DHT_REPLICATION,
- admit: Admit | None = None,
-) -> list[DiscoveredProvider]:
- """Asks the seed peers (the routers we are connected to) for providers of
- key and follows the closer peers they name for a few rounds."""
+ if resp.error:
+ raise DirectoryError(resp.error)
+ return resp
+
+
+async def request(host: IHost, routers: Iterable[ID], req: pb.DirectoryRequest) -> pb.DirectoryResponse:
+ """Sends req to the held routers in turn until one answers. A router that
+ is restarting costs the request a timeout, not an answer."""
+ last: Exception | None = None
+ asked = 0
+ for router in routers:
+ asked += 1
+ try:
+ return await _ask(host, router, req)
+ except Exception as err: # noqa: BLE001 - the next router is asked
+ last = err
+ logger.warning("directory: request through router %s failed: %s", router, err)
+ if asked == 0:
+ raise DirectoryError("no held router to ask")
+ raise DirectoryError(f"no held router answered: {last}")
+
+
+async def provide(host: IHost, routers: Iterable[ID], key: bytes, addrs: Iterable[multiaddr.Multiaddr]) -> None:
+ """Announces this member as a provider of key, reached at addrs, to a
+ held router; the router keeps the record and places it on the routers
+ closest to the key."""
+ req = pb.DirectoryRequest(provide=pb.DirectoryProvide(key=key, addrs=[str(a) for a in addrs]))
+ await request(host, routers, req)
+
+
+async def find_providers(host: IHost, routers: Iterable[ID], key: bytes, limit: int = DEFAULT_LIMIT) -> list[DiscoveredProvider]:
+ """Who provides key, up to limit, with their addresses; empty when nobody does."""
+ req = pb.DirectoryRequest(find_providers=pb.DirectoryFindProviders(key=key, limit=max(limit, 1)))
+ resp = await request(host, routers, req)
found: dict[str, DiscoveredProvider] = {}
- self_id = host.get_id()
-
- def take(resp: kad.Message) -> bool:
- for info in _peer_infos(resp.providerPeers):
- if info.peer_id == self_id:
- continue
- entry = found.setdefault(str(info.peer_id), DiscoveredProvider(peer_id=str(info.peer_id)))
- for ma in info.addrs:
- if str(ma) not in entry.addrs:
- entry.addrs.append(str(ma))
- return bool(found)
-
- await _walk(host, kad.Message(type=kad.Message.GET_PROVIDERS, key=key), seeds, take, replication=replication, admit=admit)
+ self_id = str(host.get_id())
+ for rec in resp.providers:
+ if rec.peer_id == self_id:
+ continue
+ entry = found.setdefault(rec.peer_id, DiscoveredProvider(peer_id=rec.peer_id))
+ for a in rec.addrs:
+ if a not in entry.addrs:
+ entry.addrs.append(a)
return list(found.values())[:limit]
-async def _walk(
- host: IHost,
- req: kad.Message,
- seeds: Iterable[ID],
- take: Callable[[kad.Message], bool],
- skip: frozenset[ID] | set[ID] = frozenset(),
- replication: int = DEFAULT_DHT_REPLICATION,
- admit: Admit | None = None,
-) -> None:
- """The Kademlia walk both lookups share. A round asks its peers at once
- and ends when an answer satisfies `take`; only otherwise are the closer
- peers the answers named dialed, at once as well. A round thus costs one
- query and one dial timeout, not one per peer: right after a rollout the
- routers' tables still name the pods it replaced, and the control plane
- may list a router this member cannot reach. A peer that did not answer
- its dial is not dialed again. A router names the routers closest to the
- key in order, and a record lives on the `replication` closest, so only
- that many of them are followed from one round to the next.
-
- A router serves a query only from a member that authenticated with it,
- so `admit` runs before each query. A router the walk connected to for
- the query is disconnected when the walk ends: the member keeps sessions
- with the routers it holds and with no other."""
- self_id = host.get_id()
- held = set(host.get_connected_peers())
- try:
- await _walk_rounds(host, req, seeds, take, skip, replication, admit, self_id)
- finally:
- for peer_id in set(host.get_connected_peers()) - held - {self_id}:
- try:
- await host.disconnect(peer_id)
- except Exception as err: # noqa: BLE001 - already gone
- logger.debug("dht: disconnect from %s after the walk: %s", peer_id, err)
-
-
-async def _walk_rounds(
- host: IHost,
- req: kad.Message,
- seeds: Iterable[ID],
- take: Callable[[kad.Message], bool],
- skip: frozenset[ID] | set[ID],
- replication: int,
- admit: Admit | None,
- self_id: ID,
-) -> None:
- asked: set[ID] = set()
- frontier: list[ID] = list(seeds)
- fanout = min(max(replication, 1), _MAX_PEERS_PER_ROUND)
- for _ in range(_MAX_ROUNDS):
- batch = [p for p in frontier if p not in asked and p != self_id][:_MAX_PEERS_PER_ROUND]
- if not batch:
- return
- asked.update(batch)
- answers: list[kad.Message] = []
-
- async def ask(peer_id: ID) -> None:
- if admit is not None:
- try:
- await admit(peer_id)
- except Exception as err: # noqa: BLE001 - a router that refuses us is skipped
- logger.debug("dht: %s did not admit this member: %s", peer_id, err)
- return
- resp = await _query(host, peer_id, req)
- if resp is not None:
- answers.append(resp)
-
- async with trio.open_nursery() as nursery:
- for peer_id in batch:
- nursery.start_soon(ask, peer_id)
- # Every answer is taken: the providers each router knows add up.
- satisfied = False
- for resp in answers:
- satisfied = take(resp) or satisfied
- if satisfied:
- return
-
- closer: dict[ID, PeerInfo] = {}
- for resp in answers:
- for info in _peer_infos(resp.closerPeers):
- if len(closer) >= fanout:
- break
- if info.peer_id not in asked and info.peer_id != self_id and info.peer_id not in skip:
- closer.setdefault(info.peer_id, info)
- connected = set(host.get_connected_peers())
- frontier = [p for p in closer if p in connected]
-
- async def reach(info: PeerInfo) -> None:
- try:
- await dial(host, info)
- except Exception: # noqa: BLE001 - unreachable closer peers are skipped
- asked.add(info.peer_id)
- return
- frontier.append(info.peer_id)
-
- async with trio.open_nursery() as nursery:
- for peer_id, info in closer.items():
- if peer_id not in connected:
- nursery.start_soon(reach, info)
+async def find_peer(host: IHost, routers: Iterable[ID], target: ID) -> list[multiaddr.Multiaddr]:
+ """Where target is reached: its addresses, each ending in its peer ID,
+ relayed ones included. Empty when nobody knows the peer."""
+ req = pb.DirectoryRequest(find_peer=pb.DirectoryFindPeer(peer_id=str(target)))
+ resp = await request(host, routers, req)
+ if not resp.HasField("peer") or resp.peer.peer_id != str(target):
+ return []
+ out: list[multiaddr.Multiaddr] = []
+ for a in resp.peer.addrs:
+ try:
+ out.append(multiaddr.Multiaddr(a))
+ except Exception: # noqa: BLE001 - a malformed address from a router is dropped
+ logger.debug("directory: dropping address %r of %s", a, target)
+ return out
diff --git a/sdk/python/src/agent_mesh/mesh.py b/sdk/python/src/agent_mesh/mesh.py
index a7132837..392fed4a 100644
--- a/sdk/python/src/agent_mesh/mesh.py
+++ b/sdk/python/src/agent_mesh/mesh.py
@@ -24,7 +24,6 @@ from typing import Callable, Mapping, Optional
from ._proto import agentmesh_pb2 as pb
from .controlplane import ROLE_NODE, ControlPlaneClient, Enrollment, KeysNotTrustedError, Transport
from .credential import MeshCredential, encode_auth_frame
-from .discovery import DEFAULT_DHT_REPLICATION
from .identity import Identity
_IDENTITY_FILE = "identity.key"
@@ -99,7 +98,6 @@ class AgentMesh:
self.control_plane = control_plane
self._credential = credential
self._routers: list[pb.RouterInfo] = []
- self._dht_replication = DEFAULT_DHT_REPLICATION
self._state_dir = state_dir
self._jwt_source = jwt_source
# refresh() and sync_control_plane() run in worker threads of the
@@ -120,13 +118,6 @@ class AgentMesh:
"""The router catalog (ControlPlaneInfoResponse.routers) from the most recent /info pull."""
return list(self._routers)
- @property
- def dht_replication(self) -> int:
- """How many routers hold each record and answer each lookup on this mesh,
- as /info last published it (ControlPlaneInfoResponse.directory);
- DEFAULT_DHT_REPLICATION until the control plane has answered or when it
- publishes none."""
- return self._dht_replication
def attenuate(self, rule: pb.TaskAuthorizationRule) -> MeshCredential:
"""Returns a new MeshCredential with a tar_block appended offline in memory."""
@@ -326,8 +317,6 @@ class AgentMesh:
info = self.control_plane.info()
routers = list(info.routers)
self._routers = routers
- if info.HasField("directory") and info.directory.replication >= 1:
- self._dht_replication = info.directory.replication
if info.router_addresses:
self._credential = replace(self._credential, router_addresses=list(info.router_addresses))
elif routers:
diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.py
index 6dc35aeb..73fda5a6 100644
--- a/sdk/python/src/agent_mesh/session.py
+++ b/sdk/python/src/agent_mesh/session.py
@@ -27,10 +27,8 @@ import trio
from libp2p.abc import IHost
from libp2p.peer.id import ID
from libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addr
-from libp2p.pubsub.gossipsub import PROTOCOL_ID as GOSSIPSUB_V10
-from libp2p.pubsub.gossipsub import PROTOCOL_ID_V11 as GOSSIPSUB_V11
-from libp2p.pubsub.gossipsub import PROTOCOL_ID_V12 as GOSSIPSUB_V12
-from libp2p.pubsub.gossipsub import GossipSub
+from libp2p.pubsub.floodsub import PROTOCOL_ID as FLOODSUB
+from libp2p.pubsub.floodsub import FloodSub
from libp2p.pubsub.pubsub import Pubsub
from libp2p.tools.anyio_service import background_trio_service
from mcp import ClientSession
@@ -125,13 +123,13 @@ def canonical_peer_ids(ids: Sequence[str]) -> list[str]:
class _MeshsubNoise(logging.Filter):
- """py-libp2p's pubsub opens a meshsub stream to every new peer and the host
- logs an error for each one that does not answer. Peers that leave during
- that exchange and members without pubsub are ordinary on the mesh."""
+ """py-libp2p's pubsub opens a stream to every new peer and the host logs
+ an error for each one that does not answer. Peers that leave during that
+ exchange and members without pubsub are ordinary on the mesh."""
def filter(self, record: logging.LogRecord) -> bool:
message = record.getMessage()
- return not ("Failed to open stream" in message and "/meshsub/" in message)
+ return not ("Failed to open stream" in message and ("/meshsub/" in message or "/floodsub/" in message))
_MESHSUB_NOISE = _MeshsubNoise()
@@ -577,15 +575,14 @@ class MeshSession:
return out
async def _routed_addresses(self, target: ID) -> tuple[list[multiaddr.Multiaddr], list[multiaddr.Multiaddr]]:
- """What the routers' DHT knows for a peer: its direct addresses this
+ """What the directory knows for a peer: its direct addresses this
host can dial, without the peer suffix, and the relays it reserved on
that are not admitted routers of this member; those the caller tried
already."""
direct: list[multiaddr.Multiaddr] = []
relays: list[multiaddr.Multiaddr] = []
- seeds = [ID.from_base58(r.peer_id) for r in self.routers]
suffix = f"/p2p/{target}"
- for ma in await find_peer(self.host, target, seeds, self.mesh.dht_replication, admit=self._admit_for_lookup):
+ for ma in await find_peer(self.host, self._directory_routers(), target):
text = str(ma)
if "/p2p-circuit" not in text:
try:
@@ -603,14 +600,9 @@ class MeshSession:
relays.append(relay_addr)
return direct, relays
- async def _admit_for_lookup(self, peer_id: ID) -> None:
- """Runs the handshake with a router a lookup is about to query. A
- router this member holds is authenticated already; any other serves
- the query only once this member authenticated on the connection the
- walk just opened, and the walk closes that connection when it ends."""
- if any(r.peer_id == str(peer_id) for r in self.routers):
- return
- await _authenticate_router(self.host, self.mesh, peer_id)
+ def _directory_routers(self) -> list[ID]:
+ """The routers this member holds, which answer its directory requests."""
+ return [ID.from_base58(r.peer_id) for r in self.routers]
async def _admit_router(self, addr: multiaddr.Multiaddr) -> AdmittedRouter:
"""Dials a router, runs the handshake and, its role verified, adds it
@@ -650,15 +642,14 @@ class MeshSession:
return await authenticate_with_peer(self.host, peer_id, self.mesh.auth_frame(), self.mesh.credential.control_plane_keys)
async def discover(self, service: str, name: str | None = None, limit: int = 20) -> list[DiscoveredProvider]:
- """Looks the DHT up for peers offering a service: "mcp://calc", the same
- string call_tool and request take, or a type alone ("mcp") for every
- service of that type, or (type, name). The routers we are connected to
- seed the walk."""
+ """Asks the directory for peers offering a service: "mcp://calc", the
+ same string call_tool and request take, or a type alone ("mcp") for
+ every service of that type, or (type, name). A router this member
+ holds answers."""
service_type, service_name = parse_service_target(service) if "://" in service else (service, name)
if service_type not in ("mcp", "inference", "a2a", "egress"):
raise ValueError(f"service type must be mcp, inference, a2a or egress, got {service_type!r}")
- seeds = [ID.from_base58(r.peer_id) for r in self.routers]
- return await find_providers(self.host, service_key(service_type, service_name), seeds, limit, self.mesh.dht_replication, admit=self._admit_for_lookup)
+ return await find_providers(self.host, self._directory_routers(), service_key(service_type, service_name), limit)
def open_mcp(
self,
@@ -1089,13 +1080,15 @@ async def join_mesh(
)
host.set_stream_handler(STOP_PROTOCOL, stop_stream_handler(host))
# Built before any connection: Pubsub learns of peers through a notifee it
- # registers here. StrictSign, as every Go component pins it.
- gossipsub = GossipSub(protocols=[GOSSIPSUB_V12, GOSSIPSUB_V11, GOSSIPSUB_V10], degree=6, degree_low=4, degree_high=12)
- pubsub = Pubsub(host, gossipsub, strict_signing=True)
+ # registers here. StrictSign, as every Go component pins it. A member is
+ # a leaf of the gossip: it speaks floodsub, so a router delivers every
+ # event to it outright and it takes no part in the routers' gossipsub mesh.
+ floodsub = FloodSub(protocols=[FLOODSUB])
+ pubsub = Pubsub(host, floodsub, strict_signing=True)
logging.getLogger("libp2p.host.basic_host").addFilter(_MESHSUB_NOISE)
try:
- async with host.run(listen_addrs=listen), background_trio_service(pubsub), background_trio_service(gossipsub):
+ async with host.run(listen_addrs=listen), background_trio_service(pubsub):
await pubsub.wait_until_ready()
admitted = await _admit(host, mesh, ordered, reserve, target_count=target_count)
async with trio.open_nursery() as nursery:
diff --git a/sdk/python/tests/test_discovery.py b/sdk/python/tests/test_discovery.py
index 9b7a27af..9e9d5b71 100644
--- a/sdk/python/tests/test_discovery.py
+++ b/sdk/python/tests/test_discovery.py
@@ -12,34 +12,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""The DHT walks against a fake host: what they do with the closer peers a
-router names. The wire format against a real router is covered by
-tests/integration/sdk_mesh_test.go."""
+"""The directory client: a request goes to a held router, to the next when
+that one fails, and the answer is what the router said."""
+
+import hashlib
-import multiaddr
import pytest
import trio
import trio.testing
-from libp2p.kad_dht.pb import kademlia_pb2 as kad
from libp2p.peer.id import ID
-from libp2p.peer.peerstore import PeerStore
from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes
-from agent_mesh.discovery import DHT_PROTOCOL, _QUERY_TIMEOUT, find_peer, find_providers
-from agent_mesh.host import DIAL_TIMEOUT
+from agent_mesh._proto import agentmesh_pb2 as pb
+from agent_mesh.discovery import DIRECTORY_PROTOCOL, _REQUEST_TIMEOUT, DirectoryError, find_peer, find_providers, peer_key, provide, service_key
SELF = ID.from_base58("12D3KooWJoJPhXqLrGAcVsGSzHMzWrmoHCWoTzVSSiT3JfR7RBVo")
ROUTER = ID.from_base58("12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88")
ROUTER_B = ID.from_base58("12D3KooWBTdQ3QQZztZFaxQSTzJx5ZSbpgM8zfs43VYzBXAFkdZm")
-DARK_ROUTER = ID.from_base58("12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM")
TARGET = ID.from_base58("12D3KooWQYhTNQdmr3ArTeUHRYzFg94BKyTkoWBDWez9kSCVe2Xo")
PROVIDER = ID.from_base58("12D3KooWPZCZBAZRx1TNpPt19sZ7PKcdce8rE7gNGwhzLJtWXU9u")
-GONE_POD = ID.from_base58("12D3KooWEETCw8Eb2fGFfenvEEMWhvKCe9fThJeSG8oNGrpPGp1r")
-GONE_POD_2 = ID.from_base58("12D3KooWKmPdCr3BcvPAsP5mAhraWvKwCBT3M58mWe5f7TrFHSRT")
class _Stream:
- """One kad request answered from a table, as a router's DHT would."""
+ """One directory request answered by a function, as a router would."""
def __init__(self, answer):
self._answer = answer
@@ -51,7 +46,7 @@ class _Stream:
async def read(self, n=None):
if self._req:
- req = kad.Message.FromString(await read_varint_prefixed_bytes(_Bytes(self._req)))
+ req = pb.DirectoryRequest.FromString(await read_varint_prefixed_bytes(_Bytes(self._req)))
self._req = b""
self._resp = encode_varint_prefixed(self._answer(req).SerializeToString())
n = len(self._resp) if n is None else n
@@ -73,160 +68,128 @@ class _Bytes:
class _Host:
- """Connected to its routers; any other peer a router names never
- answers a SYN."""
+ """Connected to its routers, each answering with a function; a router
+ that is None never answers a stream open."""
def __init__(self, answers):
self._answers = answers
- self.connected = set(answers)
- self.dials = []
- self.peerstore = PeerStore()
+ self.asked = []
def get_id(self):
return SELF
- def get_peerstore(self):
- return self.peerstore
-
- def get_network(self):
- return None
-
- def get_connected_peers(self):
- return list(self.connected)
-
- async def connect(self, info):
- self.dials.append(info.peer_id)
- await trio.sleep_forever()
-
async def new_stream(self, peer_id, protocols):
- assert protocols == [DHT_PROTOCOL]
- assert peer_id in self.connected
- return _Stream(self._answers[peer_id])
+ assert protocols == [DIRECTORY_PROTOCOL]
+ self.asked.append(peer_id)
+ answer = self._answers[peer_id]
+ if answer is None:
+ await trio.sleep_forever()
+ return _Stream(answer)
- async def disconnect(self, peer_id):
- self.connected.discard(peer_id)
+def _record(peer_id, *addrs):
+ return pb.PeerRecord(peer_id=str(peer_id), addrs=[f"{a}/p2p/{peer_id}" for a in addrs])
-def _peer(peer_id, *addrs):
- return kad.Message.Peer(id=peer_id.to_bytes(), addrs=[multiaddr.Multiaddr(a).to_bytes() for a in addrs])
+def test_keys_are_the_sha256_multihash_of_the_name():
+ """The same bytes api.ServiceKey and api.PeerKey produce, so a record a
+ Go member announced is found from Python and the other way round."""
+ want = b"\x12\x20" + hashlib.sha256(b"mesh:service:mcp:calculator").digest()
+ assert service_key("mcp", "calculator") == want
+ assert service_key("mcp") == b"\x12\x20" + hashlib.sha256(b"mesh:service:mcp").digest()
+ assert peer_key(TARGET) == b"\x12\x20" + hashlib.sha256(f"mesh:peer:{TARGET}".encode()).digest()
+ assert service_key("mcp") != service_key("mcp", "calculator")
-def _names_the_dark_router(req):
- return kad.Message(type=req.type, closerPeers=[_peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501")])
+def test_find_providers_asks_one_held_router_and_takes_its_answer():
+ def knows_provider(req):
+ assert req.WhichOneof("request") == "find_providers"
+ assert req.find_providers.key == service_key("mcp", "calculator")
+ return pb.DirectoryResponse(providers=[_record(PROVIDER, "/ip4/10.0.0.9/tcp/4001"), _record(SELF, "/ip4/10.0.0.1/tcp/1")])
-def test_find_providers_dials_an_unanswering_closer_peer_once():
- """A router the control plane lists may be dark from where a member runs
- (a public address a network policy drops). Every router names it as
- closer to every key; the walk dials it once, bounded, then moves on."""
- host = _Host({ROUTER: _names_the_dark_router})
+ host = _Host({ROUTER: knows_provider, ROUTER_B: knows_provider})
async def main():
- started = trio.current_time()
- assert await find_providers(host, b"key", [ROUTER]) == []
- assert host.dials == [DARK_ROUTER]
- assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)
-
- trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+ found = await find_providers(host, [ROUTER, ROUTER_B], service_key("mcp", "calculator"))
+ assert [p.peer_id for p in found] == [str(PROVIDER)], "the member itself is left out"
+ assert found[0].addrs == [f"/ip4/10.0.0.9/tcp/4001/p2p/{PROVIDER}"]
+ assert host.asked == [ROUTER], "one router answers; the second is not asked"
+ trio.run(main)
-def test_find_peer_dials_an_unanswering_closer_peer_once():
- host = _Host({ROUTER: _names_the_dark_router})
-
- async def main():
- started = trio.current_time()
- assert await find_peer(host, TARGET, [ROUTER]) == []
- assert host.dials == [DARK_ROUTER]
- assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)
-
- trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+def test_a_router_that_fails_is_skipped_for_the_next():
+ """A router that does not answer costs the request one timeout; a router
+ that answers with an error is skipped at once. Only when every held
+ router fails does the request fail."""
-def test_find_peer_returns_the_target_a_router_knows():
- def knows_target(req):
- return kad.Message(
- type=req.type,
- closerPeers=[_peer(TARGET, "/ip4/10.0.0.9/tcp/4001"), _peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501")],
- )
+ def errors(req):
+ return pb.DirectoryResponse(error="the directory is not up")
- host = _Host({ROUTER: knows_target})
+ def answers(req):
+ return pb.DirectoryResponse(providers=[_record(PROVIDER, "/ip4/10.0.0.9/tcp/4001")])
async def main():
- addrs = await find_peer(host, TARGET, [ROUTER])
- assert [str(a) for a in addrs] == ["/ip4/10.0.0.9/tcp/4001"]
- assert host.dials == []
-
- trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
-
+ host = _Host({ROUTER: None, ROUTER_B: answers})
+ started = trio.current_time()
+ found = await find_providers(host, [ROUTER, ROUTER_B], b"\x12\x20" + bytes(32))
+ assert [p.peer_id for p in found] == [str(PROVIDER)]
+ assert host.asked == [ROUTER, ROUTER_B]
+ assert trio.current_time() - started == pytest.approx(_REQUEST_TIMEOUT)
-def _after_a_rollout(req):
- """A router's answer minutes after a deploy: the provider, and a table
- that still names the pods the rollout replaced."""
- return kad.Message(
- type=req.type,
- providerPeers=[_peer(PROVIDER, "/ip4/10.84.3.81/tcp/5002")],
- closerPeers=[
- _peer(GONE_POD, "/ip4/10.84.1.7/tcp/5002"),
- _peer(GONE_POD_2, "/ip4/10.84.2.9/tcp/5002"),
- _peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501"),
- ],
- )
+ host = _Host({ROUTER: errors, ROUTER_B: answers})
+ assert [p.peer_id for p in await find_providers(host, [ROUTER, ROUTER_B], b"\x12\x20" + bytes(32))] == [str(PROVIDER)]
+ host = _Host({ROUTER: errors, ROUTER_B: errors})
+ with pytest.raises(DirectoryError, match="no held router answered"):
+ await find_providers(host, [ROUTER, ROUTER_B], b"\x12\x20" + bytes(32))
-def test_providers_in_hand_end_the_walk_before_any_closer_peer_is_dialed():
- """The probe runs right after every canary was rolled: the routers'
- tables are full of addresses nobody answers on. The first round already
- has the provider; nothing else is dialed."""
- host = _Host({ROUTER: _after_a_rollout, ROUTER_B: _after_a_rollout})
-
- async def main():
- started = trio.current_time()
- found = await find_providers(host, b"key", [ROUTER, ROUTER_B])
- assert [p.peer_id for p in found] == [str(PROVIDER)]
- assert host.dials == []
- assert trio.current_time() - started == 0
+ with pytest.raises(DirectoryError, match="no held router to ask"):
+ await find_providers(_Host({}), [], b"\x12\x20" + bytes(32))
trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
-def test_stale_closer_peers_are_dialed_at_once_not_one_after_another():
- """When nobody has the record, the walk follows the closer peers. Three
- that never answer cost one dial timeout together, not three."""
-
- def knows_nothing(req):
- return kad.Message(type=req.type, closerPeers=_after_a_rollout(req).closerPeers)
+def test_find_peer_returns_the_addresses_the_router_knows_or_nothing():
+ def knows_target(req):
+ assert req.WhichOneof("request") == "find_peer"
+ if req.find_peer.peer_id == str(TARGET):
+ return pb.DirectoryResponse(peer=_record(TARGET, "/ip4/10.0.0.9/tcp/4001", f"/ip4/10.0.0.2/tcp/4501/p2p/{ROUTER}/p2p-circuit"))
+ return pb.DirectoryResponse()
- host = _Host({ROUTER: knows_nothing, ROUTER_B: knows_nothing})
+ host = _Host({ROUTER: knows_target})
async def main():
- started = trio.current_time()
- assert await find_providers(host, b"key", [ROUTER, ROUTER_B]) == []
- assert sorted(map(str, host.dials)) == sorted(map(str, [GONE_POD, GONE_POD_2, DARK_ROUTER]))
- assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)
+ addrs = await find_peer(host, [ROUTER], TARGET)
+ assert [str(a) for a in addrs] == [
+ f"/ip4/10.0.0.9/tcp/4001/p2p/{TARGET}",
+ f"/ip4/10.0.0.2/tcp/4501/p2p/{ROUTER}/p2p-circuit/p2p/{TARGET}",
+ ]
+ assert await find_peer(host, [ROUTER], PROVIDER) == []
- trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+ trio.run(main)
-def test_the_routers_are_asked_at_once():
- """A router that takes its time to answer does not delay the others'
- answers: the round lasts as long as the slowest query, bounded."""
+def test_provide_announces_this_member_at_its_addresses():
+ seen = []
- class SlowHost(_Host):
- async def new_stream(self, peer_id, protocols):
- if peer_id == ROUTER_B:
- await trio.sleep_forever()
- return await super().new_stream(peer_id, protocols)
+ def takes(req):
+ assert req.WhichOneof("request") == "provide"
+ seen.append(req.provide)
+ return pb.DirectoryResponse()
- host = SlowHost({ROUTER: _after_a_rollout, ROUTER_B: _after_a_rollout})
+ host = _Host({ROUTER: takes})
async def main():
- started = trio.current_time()
- found = await find_providers(host, b"key", [ROUTER, ROUTER_B])
- assert [p.peer_id for p in found] == [str(PROVIDER)]
- # ROUTER_B's stream open ran to its timeout while ROUTER answered.
- assert trio.current_time() - started == pytest.approx(_QUERY_TIMEOUT)
+ import multiaddr
- trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+ await provide(host, [ROUTER], peer_key(SELF), [multiaddr.Multiaddr("/ip4/10.0.0.1/tcp/1")])
+ assert len(seen) == 1
+ assert seen[0].key == peer_key(SELF)
+ assert list(seen[0].addrs) == ["/ip4/10.0.0.1/tcp/1"]
+ assert seen[0].provider == "", "a member names no provider; it announces itself"
+
+ trio.run(main)
def test_read_bounded_varint_prefixed_bytes_rejects_oversized_prefix_before_reading_payload():
@@ -253,78 +216,3 @@ def test_read_bounded_varint_prefixed_bytes_rejects_oversized_prefix_before_read
assert bomb.payload_reads == 0
trio.run(main)
-
-
-
-def test_the_walk_follows_only_the_replication_closest_routers():
- """A router names the routers closest to the key in order, as many as it
- knows; a record lives on the mesh's replication of them, so only that
- many are followed. With the default of three the fourth is never dialed;
- with a replication of one, only the first."""
-
- def knows_nothing(req):
- return kad.Message(
- type=req.type,
- closerPeers=[
- _peer(GONE_POD, "/ip4/10.84.1.7/tcp/5002"),
- _peer(GONE_POD_2, "/ip4/10.84.2.9/tcp/5002"),
- _peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501"),
- _peer(TARGET, "/ip4/10.0.0.9/tcp/4001"),
- ],
- )
-
- async def main(replication, want):
- host = _Host({ROUTER: knows_nothing, ROUTER_B: knows_nothing})
- assert await find_providers(host, b"key", [ROUTER, ROUTER_B], replication=replication) == []
- assert sorted(map(str, host.dials)) == sorted(map(str, want))
-
- trio.run(main, 3, [GONE_POD, GONE_POD_2, DARK_ROUTER], clock=trio.testing.MockClock(autojump_threshold=0))
- trio.run(main, 1, [GONE_POD], clock=trio.testing.MockClock(autojump_threshold=0))
-
-
-def test_the_walk_authenticates_before_each_query_and_releases_what_it_connected():
- """A router serves a query only from a member that authenticated with it,
- so `admit` runs for a router before it is queried: the held router and
- the closer one the walk connects to for the record. That connection is
- the walk's alone and is closed when the walk ends; the held router's
- stays."""
-
- class ReachableHost(_Host):
- async def connect(self, info):
- self.dials.append(info.peer_id)
- self.connected.add(info.peer_id)
- self._answers[info.peer_id] = lambda req: kad.Message(type=req.type, providerPeers=[_peer(PROVIDER, "/ip4/10.0.0.9/tcp/4001")])
-
- def names_router_b(req):
- return kad.Message(type=req.type, closerPeers=[_peer(ROUTER_B, "/ip4/10.0.0.2/tcp/4501")])
-
- host = ReachableHost({ROUTER: names_router_b})
- admitted = []
-
- async def admit(peer_id):
- assert peer_id in host.connected, "admit runs on an open connection"
- admitted.append(peer_id)
-
- async def main():
- found = await find_providers(host, b"key", [ROUTER], admit=admit)
- assert [p.peer_id for p in found] == [str(PROVIDER)]
- assert admitted == [ROUTER, ROUTER_B], "every router is admitted before it is queried, in walk order"
- assert host.connected == {ROUTER}, "the router the walk connected to is released; the held one is kept"
-
- trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
-
-
-def test_a_router_that_refuses_the_handshake_is_skipped():
- """A router that does not admit this member answers nothing; the walk
- takes the other answers and moves on."""
- host = _Host({ROUTER: _after_a_rollout, ROUTER_B: _after_a_rollout})
-
- async def admit(peer_id):
- if peer_id == ROUTER_B:
- raise RuntimeError("not admitted")
-
- async def main():
- found = await find_providers(host, b"key", [ROUTER, ROUTER_B], admit=admit)
- assert [p.peer_id for p in found] == [str(PROVIDER)]
-
- trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))