Skip to content

Agent Mesh

sdk/python: dial routers and DHT peers at once, stop when the record is in hand

Antonio Ojeaa6adc422ba9f
diff --git a/sdk/python/src/agent_mesh/discovery.py b/sdk/python/src/agent_mesh/discovery.py
index bcf7a66c..08180364 100644
--- a/sdk/python/src/agent_mesh/discovery.py
+++ b/sdk/python/src/agent_mesh/discovery.py
@@ -24,7 +24,7 @@ from __future__ import annotations
import hashlib
import logging
from dataclasses import dataclass, field
-from typing import Iterable, Literal
+from typing import Callable, Iterable, Literal
import multiaddr
import trio
@@ -76,25 +76,6 @@ def parse_service_target(target: str) -> tuple[str, str]:
return scheme, name
-async def _get_providers(host: IHost, peer_id: ID, key: bytes) -> kad.Message | None:
- try:
- stream = await open_stream(host, peer_id, DHT_PROTOCOL, _QUERY_TIMEOUT)
- except Exception as err: # noqa: BLE001 - a peer that does not serve the DHT is skipped
- logger.debug("dht: %s does not answer %s: %s", peer_id, DHT_PROTOCOL, err)
- return None
- try:
- with trio.fail_after(_QUERY_TIMEOUT):
- req = kad.Message(type=kad.Message.GET_PROVIDERS, key=key)
- await stream.write(encode_varint_prefixed(req.SerializeToString()))
- resp = kad.Message.FromString(await read_varint_prefixed_bytes(stream))
- except Exception as err: # noqa: BLE001
- logger.debug("dht: query to %s failed: %s", peer_id, err)
- return None
- finally:
- await stream.close()
- return resp if resp.type == kad.Message.GET_PROVIDERS else None
-
-
def _peer_infos(peers: Iterable[kad.Message.Peer]) -> list[PeerInfo]:
out = []
for p in peers:
@@ -131,71 +112,86 @@ async def find_peer(host: IHost, target: ID, seeds: Iterable[ID]) -> list[multia
learns them: a router answers FIND_NODE for a peer it is connected to with
that peer and its addresses, its relayed one included, and names the
routers closer to it otherwise. Empty when nobody knows the peer."""
- asked: set[ID] = set()
- frontier: list[ID] = list(seeds)
- self_id = host.get_id()
- req = kad.Message(type=kad.Message.FIND_NODE, key=target.to_bytes())
- for _ in range(_MAX_ROUNDS):
- batch = [p for p in frontier if p not in asked and p != self_id][:_MAX_PEERS_PER_ROUND]
- if not batch:
- break
- frontier = []
- for peer_id in batch:
- asked.add(peer_id)
- resp = await _query(host, peer_id, req)
- if resp is None:
- continue
- for info in _peer_infos(resp.closerPeers):
- if info.peer_id == target and info.addrs:
- return list(info.addrs)
- if info.peer_id in asked or info.peer_id in (self_id, target):
- continue
- if info.peer_id not in host.get_connected_peers():
- try:
- await dial(host, info)
- except Exception: # noqa: BLE001 - unreachable closer peers are skipped
- asked.add(info.peer_id)
- continue
- frontier.append(info.peer_id)
- return []
+ found: list[multiaddr.Multiaddr] = []
+
+ def take(resp: kad.Message) -> bool:
+ for info in _peer_infos(resp.closerPeers):
+ if info.peer_id == target and info.addrs:
+ found.extend(info.addrs)
+ return True
+ return False
+
+ await _walk(host, kad.Message(type=kad.Message.FIND_NODE, key=target.to_bytes()), seeds, take, skip={target})
+ return found
async def find_providers(host: IHost, key: bytes, seeds: Iterable[ID], limit: int = 20) -> list[DiscoveredProvider]:
"""Asks the seed peers (the routers we are connected to) for providers of
key and follows the closer peers they name for a few rounds."""
found: dict[str, DiscoveredProvider] = {}
+ self_id = host.get_id()
+
+ def take(resp: kad.Message) -> bool:
+ for info in _peer_infos(resp.providerPeers):
+ if info.peer_id == self_id:
+ continue
+ entry = found.setdefault(str(info.peer_id), DiscoveredProvider(peer_id=str(info.peer_id)))
+ for ma in info.addrs:
+ if str(ma) not in entry.addrs:
+ entry.addrs.append(str(ma))
+ return bool(found)
+
+ await _walk(host, kad.Message(type=kad.Message.GET_PROVIDERS, key=key), seeds, take)
+ return list(found.values())[:limit]
+
+
+async def _walk(host: IHost, req: kad.Message, seeds: Iterable[ID], take: Callable[[kad.Message], bool], skip: frozenset[ID] | set[ID] = frozenset()) -> None:
+ """The Kademlia walk both lookups share. A round asks its peers at once
+ and ends when an answer satisfies `take`; only otherwise are the closer
+ peers the answers named dialed, at once as well. A round thus costs one
+ query and one dial timeout, not one per peer: right after a rollout the
+ routers' tables still name the pods it replaced, and the control plane
+ may list a router this member cannot reach. A peer that did not answer
+ its dial is not dialed again."""
+ self_id = host.get_id()
asked: set[ID] = set()
frontier: list[ID] = list(seeds)
- self_id = host.get_id()
for _ in range(_MAX_ROUNDS):
batch = [p for p in frontier if p not in asked and p != self_id][:_MAX_PEERS_PER_ROUND]
if not batch:
- break
- frontier = []
- for peer_id in batch:
- asked.add(peer_id)
- resp = await _get_providers(host, peer_id, key)
- if resp is None:
- continue
- for info in _peer_infos(resp.providerPeers):
- if info.peer_id == self_id:
- continue
- entry = found.setdefault(str(info.peer_id), DiscoveredProvider(peer_id=str(info.peer_id)))
- for ma in info.addrs:
- if str(ma) not in entry.addrs:
- entry.addrs.append(str(ma))
- if len(found) >= limit:
- return list(found.values())
+ return
+ asked.update(batch)
+ answers: list[kad.Message] = []
+
+ async def ask(peer_id: ID) -> None:
+ resp = await _query(host, peer_id, req)
+ if resp is not None:
+ answers.append(resp)
+
+ async with trio.open_nursery() as nursery:
+ for peer_id in batch:
+ nursery.start_soon(ask, peer_id)
+ # Every answer is taken: the providers each router knows add up.
+ if any([take(resp) for resp in answers]):
+ return
+
+ closer: dict[ID, PeerInfo] = {}
+ for resp in answers:
for info in _peer_infos(resp.closerPeers):
- if info.peer_id in asked or info.peer_id == self_id:
- continue
- if info.peer_id not in host.get_connected_peers():
- try:
- await dial(host, info)
- except Exception: # noqa: BLE001 - unreachable closer peers are skipped
- asked.add(info.peer_id)
- continue
- frontier.append(info.peer_id)
- if found:
- break
- return list(found.values())
+ if info.peer_id not in asked and info.peer_id != self_id and info.peer_id not in skip:
+ closer.setdefault(info.peer_id, info)
+ connected = set(host.get_connected_peers())
+ frontier = [p for p in closer if p in connected]
+
+ async def reach(info: PeerInfo) -> None:
+ try:
+ await dial(host, info)
+ except Exception: # noqa: BLE001 - unreachable closer peers are skipped
+ asked.add(info.peer_id)
+ return
+ frontier.append(info.peer_id)
+
+ async with trio.open_nursery() as nursery:
+ for peer_id, info in closer.items():
+ if peer_id not in connected:
+ nursery.start_soon(reach, info)
diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.py
index 34c2fe54..e9cf93e5 100644
--- a/sdk/python/src/agent_mesh/session.py
+++ b/sdk/python/src/agent_mesh/session.py
@@ -651,21 +651,37 @@ async def _admit(host: IHost, mesh: "AgentMesh", router_addrs: list[multiaddr.Mu
and sam-node do, and reserves a relay slot on the first that admits us.
A peer reserves on the first router of its own list, so a caller that
only knew one router could not reach a peer whose list started
- elsewhere; `connect` tries the relayed path through each of these."""
- admitted: list[AdmittedRouter] = []
+ elsewhere; `connect` tries the relayed path through each of these. The
+ routers are dialed at once: one the control plane lists but this member
+ cannot reach costs a dial timeout, not one per router behind it."""
+ admitted: dict[int, AdmittedRouter] = {}
failures: list[str] = []
- for addr in router_addrs:
+
+ async def admit(index: int, addr: multiaddr.Multiaddr) -> None:
try:
info = await peer_info(addr)
await dial(host, info)
credential = await _authenticate_router(host, mesh, info.peer_id)
- reservation = await reserve_relay(host, info.peer_id) if reserve and not any(r.reservation for r in admitted) else None
- admitted.append(AdmittedRouter(peer_id=str(info.peer_id), addr=addr, credential=credential, reservation=reservation))
+ admitted[index] = AdmittedRouter(peer_id=str(info.peer_id), addr=addr, credential=credential)
except Exception as err: # noqa: BLE001 - every router is tried, the summary names each failure
failures.append(f"{addr}: {err}")
- if not admitted:
+
+ async with trio.open_nursery() as nursery:
+ for index, addr in enumerate(router_addrs):
+ nursery.start_soon(admit, index, addr)
+ routers = [admitted[i] for i in sorted(admitted)]
+ if not routers:
raise RuntimeError("no router admitted this member:\n " + "\n ".join(failures))
- return admitted
+ if reserve:
+ for index, router in enumerate(routers):
+ try:
+ routers[index] = replace(router, reservation=await reserve_relay(host, ID.from_base58(router.peer_id)))
+ break
+ except Exception as err: # noqa: BLE001 - the next admitted router is asked
+ failures.append(f"{router.addr}: {err}")
+ else:
+ raise RuntimeError("no router reserved a relay slot for this member:\n " + "\n ".join(failures))
+ return routers
async def _authenticate_router(host: IHost, mesh: "AgentMesh", peer_id: ID) -> VerifiedBiscuit:
diff --git a/sdk/python/tests/test_discovery.py b/sdk/python/tests/test_discovery.py
index 6a779a49..c2bd092c 100644
--- a/sdk/python/tests/test_discovery.py
+++ b/sdk/python/tests/test_discovery.py
@@ -24,13 +24,17 @@ from libp2p.kad_dht.pb import kademlia_pb2 as kad
from libp2p.peer.id import ID
from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes
-from agent_mesh.discovery import DHT_PROTOCOL, find_peer, find_providers
+from agent_mesh.discovery import DHT_PROTOCOL, _QUERY_TIMEOUT, find_peer, find_providers
from agent_mesh.host import DIAL_TIMEOUT
SELF = ID.from_base58("12D3KooWJoJPhXqLrGAcVsGSzHMzWrmoHCWoTzVSSiT3JfR7RBVo")
ROUTER = ID.from_base58("12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88")
+ROUTER_B = ID.from_base58("12D3KooWBTdQ3QQZztZFaxQSTzJx5ZSbpgM8zfs43VYzBXAFkdZm")
DARK_ROUTER = ID.from_base58("12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM")
TARGET = ID.from_base58("12D3KooWQYhTNQdmr3ArTeUHRYzFg94BKyTkoWBDWez9kSCVe2Xo")
+PROVIDER = ID.from_base58("12D3KooWPZCZBAZRx1TNpPt19sZ7PKcdce8rE7gNGwhzLJtWXU9u")
+GONE_POD = ID.from_base58("12D3KooWEETCw8Eb2fGFfenvEEMWhvKCe9fThJeSG8oNGrpPGp1r")
+GONE_POD_2 = ID.from_base58("12D3KooWKmPdCr3BcvPAsP5mAhraWvKwCBT3M58mWe5f7TrFHSRT")
class _Stream:
@@ -68,12 +72,12 @@ class _Bytes:
class _Host:
- """Connected to one router; the other one the router names never
+ """Connected to its routers; any other peer a router names never
answers a SYN."""
def __init__(self, answers):
self._answers = answers
- self.connected = {ROUTER}
+ self.connected = set(answers)
self.dials = []
def get_id(self):
@@ -145,3 +149,73 @@ def test_find_peer_returns_the_target_a_router_knows():
assert host.dials == []
trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+
+
+def _after_a_rollout(req):
+ """A router's answer minutes after a deploy: the provider, and a table
+ that still names the pods the rollout replaced."""
+ return kad.Message(
+ type=req.type,
+ providerPeers=[_peer(PROVIDER, "/ip4/10.84.3.81/tcp/5002")],
+ closerPeers=[
+ _peer(GONE_POD, "/ip4/10.84.1.7/tcp/5002"),
+ _peer(GONE_POD_2, "/ip4/10.84.2.9/tcp/5002"),
+ _peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501"),
+ ],
+ )
+
+
+def test_providers_in_hand_end_the_walk_before_any_closer_peer_is_dialed():
+ """The probe runs right after every canary was rolled: the routers'
+ tables are full of addresses nobody answers on. The first round already
+ has the provider; nothing else is dialed."""
+ host = _Host({ROUTER: _after_a_rollout, ROUTER_B: _after_a_rollout})
+
+ async def main():
+ started = trio.current_time()
+ found = await find_providers(host, b"key", [ROUTER, ROUTER_B])
+ assert [p.peer_id for p in found] == [str(PROVIDER)]
+ assert host.dials == []
+ assert trio.current_time() - started == 0
+
+ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+
+
+def test_stale_closer_peers_are_dialed_at_once_not_one_after_another():
+ """When nobody has the record, the walk follows the closer peers. Three
+ that never answer cost one dial timeout together, not three."""
+
+ def knows_nothing(req):
+ return kad.Message(type=req.type, closerPeers=_after_a_rollout(req).closerPeers)
+
+ host = _Host({ROUTER: knows_nothing, ROUTER_B: knows_nothing})
+
+ async def main():
+ started = trio.current_time()
+ assert await find_providers(host, b"key", [ROUTER, ROUTER_B]) == []
+ assert sorted(map(str, host.dials)) == sorted(map(str, [GONE_POD, GONE_POD_2, DARK_ROUTER]))
+ assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)
+
+ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+
+
+def test_the_routers_are_asked_at_once():
+ """A router that takes its time to answer does not delay the others'
+ answers: the round lasts as long as the slowest query, bounded."""
+
+ class SlowHost(_Host):
+ async def new_stream(self, peer_id, protocols):
+ if peer_id == ROUTER_B:
+ await trio.sleep_forever()
+ return await super().new_stream(peer_id, protocols)
+
+ host = SlowHost({ROUTER: _after_a_rollout, ROUTER_B: _after_a_rollout})
+
+ async def main():
+ started = trio.current_time()
+ found = await find_providers(host, b"key", [ROUTER, ROUTER_B])
+ assert [p.peer_id for p in found] == [str(PROVIDER)]
+ # ROUTER_B's stream open ran to its timeout while ROUTER answered.
+ assert trio.current_time() - started == pytest.approx(_QUERY_TIMEOUT)
+
+ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
diff --git a/sdk/python/tests/test_session.py b/sdk/python/tests/test_session.py
index 15e7cc00..3ab1229c 100644
--- a/sdk/python/tests/test_session.py
+++ b/sdk/python/tests/test_session.py
@@ -24,6 +24,7 @@ import biscuit_auth as ba
import multiaddr
import pytest
import trio
+import trio.testing
from libp2p import new_host
from libp2p.crypto.ed25519 import create_new_key_pair
from libp2p.custom_types import TProtocol
@@ -258,6 +259,45 @@ def test_join_admits_every_router_and_reserves_on_the_first():
trio.run(with_timeout, 30, main)
+def test_a_router_nobody_answers_costs_join_one_dial_timeout(monkeypatch):
+ """The control plane may list a router this member cannot reach (a
+ public address a network policy drops). Every router is dialed at once,
+ so join ends at DIAL_TIMEOUT whatever the dark one's position, and the
+ reservation still goes to the first *admitted* router of the list."""
+ from agent_mesh import session as session_module
+ from agent_mesh.host import DIAL_TIMEOUT
+
+ dark = "/ip4/203.0.113.7/tcp/4501/p2p/12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM"
+ router_a = "/ip4/10.0.0.1/tcp/4501/p2p/12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88"
+ router_b = "/ip4/10.0.0.2/tcp/4501/p2p/12D3KooWBTdQ3QQZztZFaxQSTzJx5ZSbpgM8zfs43VYzBXAFkdZm"
+ reserved = []
+
+ class Host:
+ async def connect(self, info):
+ if str(info.addrs[0]).startswith("/ip4/203.0.113.7/"):
+ await trio.sleep_forever()
+
+ async def authenticated(host, mesh, peer_id):
+ return f"credential of {peer_id}"
+
+ async def reserve(host, peer_id):
+ reserved.append(str(peer_id))
+ return circuit.Reservation()
+
+ monkeypatch.setattr(session_module, "_authenticate_router", authenticated)
+ monkeypatch.setattr(session_module, "reserve_relay", reserve)
+
+ async def main():
+ started = trio.current_time()
+ routers = await session_module._admit(Host(), None, [multiaddr.Multiaddr(a) for a in (dark, router_a, router_b)], reserve=True)
+ assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)
+ assert [r.peer_id[-6:] for r in routers] == ["UNCk88", "AFkdZm"]
+ assert [r.reservation is not None for r in routers] == [True, False]
+ assert [p[-6:] for p in reserved] == ["UNCk88"]
+
+ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+
+
def test_a_dropped_router_connection_is_reserved_again_before_the_ttl():
"""A router restart takes the reservation with the connection. The member
notices within the check interval and reserves again, authenticated