Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
sdk/python: dial routers and DHT peers at once, stop when the record is in hand
Antonio Ojeaa6adc422ba9f
diff --git a/sdk/python/src/agent_mesh/discovery.py b/sdk/python/src/agent_mesh/discovery.pyindex bcf7a66c..08180364 100644--- a/sdk/python/src/agent_mesh/discovery.py+++ b/sdk/python/src/agent_mesh/discovery.py@@ -24,7 +24,7 @@ from __future__ import annotationsimport hashlibimport loggingfrom dataclasses import dataclass, field-from typing import Iterable, Literal+from typing import Callable, Iterable, Literalimport multiaddrimport trio@@ -76,25 +76,6 @@ def parse_service_target(target: str) -> tuple[str, str]:return scheme, name-async def _get_providers(host: IHost, peer_id: ID, key: bytes) -> kad.Message | None:- try:- stream = await open_stream(host, peer_id, DHT_PROTOCOL, _QUERY_TIMEOUT)- except Exception as err: # noqa: BLE001 - a peer that does not serve the DHT is skipped- logger.debug("dht: %s does not answer %s: %s", peer_id, DHT_PROTOCOL, err)- return None- try:- with trio.fail_after(_QUERY_TIMEOUT):- req = kad.Message(type=kad.Message.GET_PROVIDERS, key=key)- await stream.write(encode_varint_prefixed(req.SerializeToString()))- resp = kad.Message.FromString(await read_varint_prefixed_bytes(stream))- except Exception as err: # noqa: BLE001- logger.debug("dht: query to %s failed: %s", peer_id, err)- return None- finally:- await stream.close()- return resp if resp.type == kad.Message.GET_PROVIDERS else None--def _peer_infos(peers: Iterable[kad.Message.Peer]) -> list[PeerInfo]:out = []for p in peers:@@ -131,71 +112,86 @@ async def find_peer(host: IHost, target: ID, seeds: Iterable[ID]) -> list[multialearns them: a router answers FIND_NODE for a peer it is connected to withthat peer and its addresses, its relayed one included, and names therouters closer to it otherwise. Empty when nobody knows the peer."""- asked: set[ID] = set()- frontier: list[ID] = list(seeds)- self_id = host.get_id()- req = kad.Message(type=kad.Message.FIND_NODE, key=target.to_bytes())- for _ in range(_MAX_ROUNDS):- batch = [p for p in frontier if p not in asked and p != self_id][:_MAX_PEERS_PER_ROUND]- if not batch:- break- frontier = []- for peer_id in batch:- asked.add(peer_id)- resp = await _query(host, peer_id, req)- if resp is None:- continue- for info in _peer_infos(resp.closerPeers):- if info.peer_id == target and info.addrs:- return list(info.addrs)- if info.peer_id in asked or info.peer_id in (self_id, target):- continue- if info.peer_id not in host.get_connected_peers():- try:- await dial(host, info)- except Exception: # noqa: BLE001 - unreachable closer peers are skipped- asked.add(info.peer_id)- continue- frontier.append(info.peer_id)- return []+ found: list[multiaddr.Multiaddr] = []++ def take(resp: kad.Message) -> bool:+ for info in _peer_infos(resp.closerPeers):+ if info.peer_id == target and info.addrs:+ found.extend(info.addrs)+ return True+ return False++ await _walk(host, kad.Message(type=kad.Message.FIND_NODE, key=target.to_bytes()), seeds, take, skip={target})+ return foundasync def find_providers(host: IHost, key: bytes, seeds: Iterable[ID], limit: int = 20) -> list[DiscoveredProvider]:"""Asks the seed peers (the routers we are connected to) for providers ofkey and follows the closer peers they name for a few rounds."""found: dict[str, DiscoveredProvider] = {}+ self_id = host.get_id()++ def take(resp: kad.Message) -> bool:+ for info in _peer_infos(resp.providerPeers):+ if info.peer_id == self_id:+ continue+ entry = found.setdefault(str(info.peer_id), DiscoveredProvider(peer_id=str(info.peer_id)))+ for ma in info.addrs:+ if str(ma) not in entry.addrs:+ entry.addrs.append(str(ma))+ return bool(found)++ await _walk(host, kad.Message(type=kad.Message.GET_PROVIDERS, key=key), seeds, take)+ return list(found.values())[:limit]+++async def _walk(host: IHost, req: kad.Message, seeds: Iterable[ID], take: Callable[[kad.Message], bool], skip: frozenset[ID] | set[ID] = frozenset()) -> None:+ """The Kademlia walk both lookups share. A round asks its peers at once+ and ends when an answer satisfies `take`; only otherwise are the closer+ peers the answers named dialed, at once as well. A round thus costs one+ query and one dial timeout, not one per peer: right after a rollout the+ routers' tables still name the pods it replaced, and the control plane+ may list a router this member cannot reach. A peer that did not answer+ its dial is not dialed again."""+ self_id = host.get_id()asked: set[ID] = set()frontier: list[ID] = list(seeds)- self_id = host.get_id()for _ in range(_MAX_ROUNDS):batch = [p for p in frontier if p not in asked and p != self_id][:_MAX_PEERS_PER_ROUND]if not batch:- break- frontier = []- for peer_id in batch:- asked.add(peer_id)- resp = await _get_providers(host, peer_id, key)- if resp is None:- continue- for info in _peer_infos(resp.providerPeers):- if info.peer_id == self_id:- continue- entry = found.setdefault(str(info.peer_id), DiscoveredProvider(peer_id=str(info.peer_id)))- for ma in info.addrs:- if str(ma) not in entry.addrs:- entry.addrs.append(str(ma))- if len(found) >= limit:- return list(found.values())+ return+ asked.update(batch)+ answers: list[kad.Message] = []++ async def ask(peer_id: ID) -> None:+ resp = await _query(host, peer_id, req)+ if resp is not None:+ answers.append(resp)++ async with trio.open_nursery() as nursery:+ for peer_id in batch:+ nursery.start_soon(ask, peer_id)+ # Every answer is taken: the providers each router knows add up.+ if any([take(resp) for resp in answers]):+ return++ closer: dict[ID, PeerInfo] = {}+ for resp in answers:for info in _peer_infos(resp.closerPeers):- if info.peer_id in asked or info.peer_id == self_id:- continue- if info.peer_id not in host.get_connected_peers():- try:- await dial(host, info)- except Exception: # noqa: BLE001 - unreachable closer peers are skipped- asked.add(info.peer_id)- continue- frontier.append(info.peer_id)- if found:- break- return list(found.values())+ if info.peer_id not in asked and info.peer_id != self_id and info.peer_id not in skip:+ closer.setdefault(info.peer_id, info)+ connected = set(host.get_connected_peers())+ frontier = [p for p in closer if p in connected]++ async def reach(info: PeerInfo) -> None:+ try:+ await dial(host, info)+ except Exception: # noqa: BLE001 - unreachable closer peers are skipped+ asked.add(info.peer_id)+ return+ frontier.append(info.peer_id)++ async with trio.open_nursery() as nursery:+ for peer_id, info in closer.items():+ if peer_id not in connected:+ nursery.start_soon(reach, info)diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.pyindex 34c2fe54..e9cf93e5 100644--- a/sdk/python/src/agent_mesh/session.py+++ b/sdk/python/src/agent_mesh/session.py@@ -651,21 +651,37 @@ async def _admit(host: IHost, mesh: "AgentMesh", router_addrs: list[multiaddr.Muand sam-node do, and reserves a relay slot on the first that admits us.A peer reserves on the first router of its own list, so a caller thatonly knew one router could not reach a peer whose list started- elsewhere; `connect` tries the relayed path through each of these."""- admitted: list[AdmittedRouter] = []+ elsewhere; `connect` tries the relayed path through each of these. The+ routers are dialed at once: one the control plane lists but this member+ cannot reach costs a dial timeout, not one per router behind it."""+ admitted: dict[int, AdmittedRouter] = {}failures: list[str] = []- for addr in router_addrs:++ async def admit(index: int, addr: multiaddr.Multiaddr) -> None:try:info = await peer_info(addr)await dial(host, info)credential = await _authenticate_router(host, mesh, info.peer_id)- reservation = await reserve_relay(host, info.peer_id) if reserve and not any(r.reservation for r in admitted) else None- admitted.append(AdmittedRouter(peer_id=str(info.peer_id), addr=addr, credential=credential, reservation=reservation))+ admitted[index] = AdmittedRouter(peer_id=str(info.peer_id), addr=addr, credential=credential)except Exception as err: # noqa: BLE001 - every router is tried, the summary names each failurefailures.append(f"{addr}: {err}")- if not admitted:++ async with trio.open_nursery() as nursery:+ for index, addr in enumerate(router_addrs):+ nursery.start_soon(admit, index, addr)+ routers = [admitted[i] for i in sorted(admitted)]+ if not routers:raise RuntimeError("no router admitted this member:\n " + "\n ".join(failures))- return admitted+ if reserve:+ for index, router in enumerate(routers):+ try:+ routers[index] = replace(router, reservation=await reserve_relay(host, ID.from_base58(router.peer_id)))+ break+ except Exception as err: # noqa: BLE001 - the next admitted router is asked+ failures.append(f"{router.addr}: {err}")+ else:+ raise RuntimeError("no router reserved a relay slot for this member:\n " + "\n ".join(failures))+ return routersasync def _authenticate_router(host: IHost, mesh: "AgentMesh", peer_id: ID) -> VerifiedBiscuit:diff --git a/sdk/python/tests/test_discovery.py b/sdk/python/tests/test_discovery.pyindex 6a779a49..c2bd092c 100644--- a/sdk/python/tests/test_discovery.py+++ b/sdk/python/tests/test_discovery.py@@ -24,13 +24,17 @@ from libp2p.kad_dht.pb import kademlia_pb2 as kadfrom libp2p.peer.id import IDfrom libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes-from agent_mesh.discovery import DHT_PROTOCOL, find_peer, find_providers+from agent_mesh.discovery import DHT_PROTOCOL, _QUERY_TIMEOUT, find_peer, find_providersfrom agent_mesh.host import DIAL_TIMEOUTSELF = ID.from_base58("12D3KooWJoJPhXqLrGAcVsGSzHMzWrmoHCWoTzVSSiT3JfR7RBVo")ROUTER = ID.from_base58("12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88")+ROUTER_B = ID.from_base58("12D3KooWBTdQ3QQZztZFaxQSTzJx5ZSbpgM8zfs43VYzBXAFkdZm")DARK_ROUTER = ID.from_base58("12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM")TARGET = ID.from_base58("12D3KooWQYhTNQdmr3ArTeUHRYzFg94BKyTkoWBDWez9kSCVe2Xo")+PROVIDER = ID.from_base58("12D3KooWPZCZBAZRx1TNpPt19sZ7PKcdce8rE7gNGwhzLJtWXU9u")+GONE_POD = ID.from_base58("12D3KooWEETCw8Eb2fGFfenvEEMWhvKCe9fThJeSG8oNGrpPGp1r")+GONE_POD_2 = ID.from_base58("12D3KooWKmPdCr3BcvPAsP5mAhraWvKwCBT3M58mWe5f7TrFHSRT")class _Stream:@@ -68,12 +72,12 @@ class _Bytes:class _Host:- """Connected to one router; the other one the router names never+ """Connected to its routers; any other peer a router names neveranswers a SYN."""def __init__(self, answers):self._answers = answers- self.connected = {ROUTER}+ self.connected = set(answers)self.dials = []def get_id(self):@@ -145,3 +149,73 @@ def test_find_peer_returns_the_target_a_router_knows():assert host.dials == []trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))+++def _after_a_rollout(req):+ """A router's answer minutes after a deploy: the provider, and a table+ that still names the pods the rollout replaced."""+ return kad.Message(+ type=req.type,+ providerPeers=[_peer(PROVIDER, "/ip4/10.84.3.81/tcp/5002")],+ closerPeers=[+ _peer(GONE_POD, "/ip4/10.84.1.7/tcp/5002"),+ _peer(GONE_POD_2, "/ip4/10.84.2.9/tcp/5002"),+ _peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501"),+ ],+ )+++def test_providers_in_hand_end_the_walk_before_any_closer_peer_is_dialed():+ """The probe runs right after every canary was rolled: the routers'+ tables are full of addresses nobody answers on. The first round already+ has the provider; nothing else is dialed."""+ host = _Host({ROUTER: _after_a_rollout, ROUTER_B: _after_a_rollout})++ async def main():+ started = trio.current_time()+ found = await find_providers(host, b"key", [ROUTER, ROUTER_B])+ assert [p.peer_id for p in found] == [str(PROVIDER)]+ assert host.dials == []+ assert trio.current_time() - started == 0++ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))+++def test_stale_closer_peers_are_dialed_at_once_not_one_after_another():+ """When nobody has the record, the walk follows the closer peers. Three+ that never answer cost one dial timeout together, not three."""++ def knows_nothing(req):+ return kad.Message(type=req.type, closerPeers=_after_a_rollout(req).closerPeers)++ host = _Host({ROUTER: knows_nothing, ROUTER_B: knows_nothing})++ async def main():+ started = trio.current_time()+ assert await find_providers(host, b"key", [ROUTER, ROUTER_B]) == []+ assert sorted(map(str, host.dials)) == sorted(map(str, [GONE_POD, GONE_POD_2, DARK_ROUTER]))+ assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)++ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))+++def test_the_routers_are_asked_at_once():+ """A router that takes its time to answer does not delay the others'+ answers: the round lasts as long as the slowest query, bounded."""++ class SlowHost(_Host):+ async def new_stream(self, peer_id, protocols):+ if peer_id == ROUTER_B:+ await trio.sleep_forever()+ return await super().new_stream(peer_id, protocols)++ host = SlowHost({ROUTER: _after_a_rollout, ROUTER_B: _after_a_rollout})++ async def main():+ started = trio.current_time()+ found = await find_providers(host, b"key", [ROUTER, ROUTER_B])+ assert [p.peer_id for p in found] == [str(PROVIDER)]+ # ROUTER_B's stream open ran to its timeout while ROUTER answered.+ assert trio.current_time() - started == pytest.approx(_QUERY_TIMEOUT)++ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))diff --git a/sdk/python/tests/test_session.py b/sdk/python/tests/test_session.pyindex 15e7cc00..3ab1229c 100644--- a/sdk/python/tests/test_session.py+++ b/sdk/python/tests/test_session.py@@ -24,6 +24,7 @@ import biscuit_auth as baimport multiaddrimport pytestimport trio+import trio.testingfrom libp2p import new_hostfrom libp2p.crypto.ed25519 import create_new_key_pairfrom libp2p.custom_types import TProtocol@@ -258,6 +259,45 @@ def test_join_admits_every_router_and_reserves_on_the_first():trio.run(with_timeout, 30, main)+def test_a_router_nobody_answers_costs_join_one_dial_timeout(monkeypatch):+ """The control plane may list a router this member cannot reach (a+ public address a network policy drops). Every router is dialed at once,+ so join ends at DIAL_TIMEOUT whatever the dark one's position, and the+ reservation still goes to the first *admitted* router of the list."""+ from agent_mesh import session as session_module+ from agent_mesh.host import DIAL_TIMEOUT++ dark = "/ip4/203.0.113.7/tcp/4501/p2p/12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM"+ router_a = "/ip4/10.0.0.1/tcp/4501/p2p/12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88"+ router_b = "/ip4/10.0.0.2/tcp/4501/p2p/12D3KooWBTdQ3QQZztZFaxQSTzJx5ZSbpgM8zfs43VYzBXAFkdZm"+ reserved = []++ class Host:+ async def connect(self, info):+ if str(info.addrs[0]).startswith("/ip4/203.0.113.7/"):+ await trio.sleep_forever()++ async def authenticated(host, mesh, peer_id):+ return f"credential of {peer_id}"++ async def reserve(host, peer_id):+ reserved.append(str(peer_id))+ return circuit.Reservation()++ monkeypatch.setattr(session_module, "_authenticate_router", authenticated)+ monkeypatch.setattr(session_module, "reserve_relay", reserve)++ async def main():+ started = trio.current_time()+ routers = await session_module._admit(Host(), None, [multiaddr.Multiaddr(a) for a in (dark, router_a, router_b)], reserve=True)+ assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)+ assert [r.peer_id[-6:] for r in routers] == ["UNCk88", "AFkdZm"]+ assert [r.reservation is not None for r in routers] == [True, False]+ assert [p[-6:] for p in reserved] == ["UNCk88"]++ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))++def test_a_dropped_router_connection_is_reserved_again_before_the_ttl():"""A router restart takes the reservation with the connection. The membernotices within the check interval and reserves again, authenticated