Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
sdk/python: bound every dial the DHT walk makes
Antonio Ojeaa18bc97d10b9
diff --git a/.github/k8s/sam-sdk-probe-cronjob-template.yaml b/.github/k8s/sam-sdk-probe-cronjob-template.yamlindex 0a84389a..83ddb5ec 100644--- a/.github/k8s/sam-sdk-probe-cronjob-template.yaml+++ b/.github/k8s/sam-sdk-probe-cronjob-template.yaml@@ -236,11 +236,13 @@ data:attempts=0while :; doattempts=$((attempts + 1))- out=$($program "$@" 2>&1); rc=$?+ # A program that hangs is a failure to report, not a job to let the deadline kill silently.+ out=$(timeout -k 5 120 $program "$@" 2>&1); rc=$?if [ $rc -eq 0 ] && printf '%s' "$out" | grep -q "$want"; thenPROVIDER=$(printf '%s' "$out" | grep -o 'is served by [^ ]*' | tail -1 | cut -d' ' -f4)return 0fi+ [ $rc -eq 124 ] && out="$out (hung for 120s)"[ $(( $(date +%s) - stage_start )) -ge 120 ] && fail "$stage" "no answer with '$want' in 120s ($attempts attempts; last: $(printf '%s' "$out" | tail -c 200))"sleep 5donediff --git a/sdk/python/src/agent_mesh/discovery.py b/sdk/python/src/agent_mesh/discovery.pyindex e887ae73..bcf7a66c 100644--- a/sdk/python/src/agent_mesh/discovery.py+++ b/sdk/python/src/agent_mesh/discovery.py@@ -35,7 +35,7 @@ from libp2p.peer.id import IDfrom libp2p.peer.peerinfo import PeerInfofrom libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes-from .host import open_stream+from .host import dial, open_streamlogger = logging.getLogger("agent_mesh")@@ -152,8 +152,9 @@ async def find_peer(host: IHost, target: ID, seeds: Iterable[ID]) -> list[multiacontinueif info.peer_id not in host.get_connected_peers():try:- await host.connect(info)+ await dial(host, info)except Exception: # noqa: BLE001 - unreachable closer peers are skipped+ asked.add(info.peer_id)continuefrontier.append(info.peer_id)return []@@ -190,8 +191,9 @@ async def find_providers(host: IHost, key: bytes, seeds: Iterable[ID], limit: incontinueif info.peer_id not in host.get_connected_peers():try:- await host.connect(info)+ await dial(host, info)except Exception: # noqa: BLE001 - unreachable closer peers are skipped+ asked.add(info.peer_id)continuefrontier.append(info.peer_id)if found:diff --git a/sdk/python/src/agent_mesh/host.py b/sdk/python/src/agent_mesh/host.pyindex 1576b375..1482575f 100644--- a/sdk/python/src/agent_mesh/host.py+++ b/sdk/python/src/agent_mesh/host.py@@ -122,6 +122,24 @@ def _dial_websockets_by_name(transport: WebsocketTransport) -> None:transport.dial = dial # type: ignore[method-assign]+# sam-node's swarm dial timeout. py-libp2p has none of its own: a SYN to an+# address nobody answers waits on the kernel, about two minutes, and is then+# retried, and a provider record can name a pod a rollout just replaced. A+# router the control plane lists may be unreachable from where a member runs+# (a public address a network policy drops), and the routers' DHT names it as+# a closer peer to every lookup.+DIAL_TIMEOUT = 15.0+++async def dial(host: IHost, info: PeerInfo) -> None:+ """host.connect, bounded by DIAL_TIMEOUT."""+ try:+ with trio.fail_after(DIAL_TIMEOUT):+ await host.connect(info)+ except trio.TooSlowError:+ raise ConnectionError(f"no connection to {info.peer_id} within {DIAL_TIMEOUT:g}s") from None++async def open_stream(host: IHost, peer_id: ID, protocol: TProtocol, timeout: float) -> INetStream:"""host.new_stream bounded by a timeout. py-libp2p bounds the protocolnegotiation but not the muxer, and a muxer that cannot open a stream@@ -133,7 +151,9 @@ async def open_stream(host: IHost, peer_id: ID, protocol: TProtocol, timeout: flwith trio.fail_after(timeout):return await host.new_stream(peer_id, [protocol])except trio.TooSlowError:- with trio.CancelScope(shield=True):+ # Closing writes a GO_AWAY on the very connection that stalled; give+ # it a moment, then let the peer time the socket out on its own.+ with trio.CancelScope(shield=True), trio.move_on_after(timeout):try:await host.disconnect(peer_id)except Exception: # noqa: BLE001 - already gonediff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.pyindex 2aa52c90..34c2fe54 100644--- a/sdk/python/src/agent_mesh/session.py+++ b/sdk/python/src/agent_mesh/session.py@@ -41,7 +41,7 @@ from .auth import AUTH_PROTOCOL, auth_stream_handler, authenticate_with_peerfrom .authorizer import ProviderAuthorizerOptionsfrom .biscuit import ROLE_ROUTER, VerifiedBiscuit, require_rolefrom .discovery import DiscoveredProvider, find_peer, find_providers, parse_service_target, service_key-from .host import create_mesh_host, dial_addrs, peer_info+from .host import create_mesh_host, dial, dial_addrs, peer_infofrom .httpx_transport import MESH_PATH_PREFIXfrom .identity import canonical_peer_idfrom .libp2p_http import (@@ -82,11 +82,6 @@ DEFAULT_CONTROL_PLANE_SYNC = 15 * 60.0FIRST_CONTROL_PLANE_SYNC = 2.0DEFAULT_CONTROL_PLANE_SYNC_JITTER = 2.0-# sam-node's swarm dial timeout. py-libp2p has none of its own: a SYN to an-# address nobody answers waits on the kernel, about two minutes, and is then-# retried, and a provider record can name a pod a rollout just replaced.-DIAL_TIMEOUT = 15.0-# How a caller names the peer it wants to reach: a provider `discover` returned,# a peer id, or a multiaddr. For a provider or a peer id the SDK dials the# addresses the peer advertised and then the relayed path through every router@@ -100,15 +95,6 @@ def parse_peer_id(text: str) -> ID:return ID.from_base58(canonical_peer_id(text))-async def dial(host: IHost, info: PeerInfo) -> None:- """host.connect, bounded by DIAL_TIMEOUT."""- try:- with trio.fail_after(DIAL_TIMEOUT):- await host.connect(info)- except trio.TooSlowError:- raise ConnectionError(f"no connection to {info.peer_id} within {DIAL_TIMEOUT:g}s") from None--def canonical_peer_ids(ids: Sequence[str]) -> list[str]:"""Canonicalizes a list from the control plane, dropping entries that arenot peer IDs: they can match nothing, so they ban nothing."""diff --git a/sdk/python/tests/test_discovery.py b/sdk/python/tests/test_discovery.pynew file mode 100644index 00000000..6a779a49--- /dev/null+++ b/sdk/python/tests/test_discovery.py@@ -0,0 +1,147 @@+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""The DHT walks against a fake host: what they do with the closer peers a+router names. The wire format against a real router is covered by+tests/integration/sdk_mesh_test.go."""++import multiaddr+import pytest+import trio+import trio.testing+from libp2p.kad_dht.pb import kademlia_pb2 as kad+from libp2p.peer.id import ID+from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes++from agent_mesh.discovery import DHT_PROTOCOL, find_peer, find_providers+from agent_mesh.host import DIAL_TIMEOUT++SELF = ID.from_base58("12D3KooWJoJPhXqLrGAcVsGSzHMzWrmoHCWoTzVSSiT3JfR7RBVo")+ROUTER = ID.from_base58("12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88")+DARK_ROUTER = ID.from_base58("12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM")+TARGET = ID.from_base58("12D3KooWQYhTNQdmr3ArTeUHRYzFg94BKyTkoWBDWez9kSCVe2Xo")+++class _Stream:+ """One kad request answered from a table, as a router's DHT would."""++ def __init__(self, answer):+ self._answer = answer+ self._req = b""+ self._resp = b""++ async def write(self, data):+ self._req += data++ async def read(self, n=None):+ if self._req:+ req = kad.Message.FromString(await read_varint_prefixed_bytes(_Bytes(self._req)))+ self._req = b""+ self._resp = encode_varint_prefixed(self._answer(req).SerializeToString())+ n = len(self._resp) if n is None else n+ out, self._resp = self._resp[:n], self._resp[n:]+ return out++ async def close(self):+ pass+++class _Bytes:+ def __init__(self, data):+ self._data = data++ async def read(self, n=None):+ n = len(self._data) if n is None else n+ out, self._data = self._data[:n], self._data[n:]+ return out+++class _Host:+ """Connected to one router; the other one the router names never+ answers a SYN."""++ def __init__(self, answers):+ self._answers = answers+ self.connected = {ROUTER}+ self.dials = []++ def get_id(self):+ return SELF++ def get_connected_peers(self):+ return list(self.connected)++ async def connect(self, info):+ self.dials.append(info.peer_id)+ await trio.sleep_forever()++ async def new_stream(self, peer_id, protocols):+ assert protocols == [DHT_PROTOCOL]+ assert peer_id in self.connected+ return _Stream(self._answers[peer_id])++ async def disconnect(self, peer_id):+ self.connected.discard(peer_id)+++def _peer(peer_id, *addrs):+ return kad.Message.Peer(id=peer_id.to_bytes(), addrs=[multiaddr.Multiaddr(a).to_bytes() for a in addrs])+++def _names_the_dark_router(req):+ return kad.Message(type=req.type, closerPeers=[_peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501")])+++def test_find_providers_dials_an_unanswering_closer_peer_once():+ """A router the control plane lists may be dark from where a member runs+ (a public address a network policy drops). Every router names it as+ closer to every key; the walk dials it once, bounded, then moves on."""+ host = _Host({ROUTER: _names_the_dark_router})++ async def main():+ started = trio.current_time()+ assert await find_providers(host, b"key", [ROUTER]) == []+ assert host.dials == [DARK_ROUTER]+ assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)++ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))+++def test_find_peer_dials_an_unanswering_closer_peer_once():+ host = _Host({ROUTER: _names_the_dark_router})++ async def main():+ started = trio.current_time()+ assert await find_peer(host, TARGET, [ROUTER]) == []+ assert host.dials == [DARK_ROUTER]+ assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)++ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))+++def test_find_peer_returns_the_target_a_router_knows():+ def knows_target(req):+ return kad.Message(+ type=req.type,+ closerPeers=[_peer(TARGET, "/ip4/10.0.0.9/tcp/4001"), _peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501")],+ )++ host = _Host({ROUTER: knows_target})++ async def main():+ addrs = await find_peer(host, TARGET, [ROUTER])+ assert [str(a) for a in addrs] == ["/ip4/10.0.0.9/tcp/4001"]+ assert host.dials == []++ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))diff --git a/sdk/python/tests/test_host.py b/sdk/python/tests/test_host.pyindex 1770a96c..7da7360b 100644--- a/sdk/python/tests/test_host.py+++ b/sdk/python/tests/test_host.py@@ -32,9 +32,8 @@ from libp2p.stream_muxer.yamux.yamux import FLAG_SYN, YAMUX_HEADER_FORMATfrom libp2p.transport.websocket.transport import WebsocketTransportfrom multiaddr.resolvers import DNSResolver-from agent_mesh.host import create_mesh_host, dial_addrs, open_stream, peer_info+from agent_mesh.host import DIAL_TIMEOUT, create_mesh_host, dial, dial_addrs, open_stream, peer_infofrom agent_mesh.identity import Identity-from agent_mesh.session import DIAL_TIMEOUT, dialROUTER = "12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88"OTHER = "12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM"@@ -274,6 +273,27 @@ def test_open_stream_ends_at_its_timeout():trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))+def test_hanging_up_on_a_stalled_connection_is_bounded_too():+ """Closing the connection writes a GO_AWAY on the very connection whose+ SYN stalled; on a blackholed peer that write parks as well. The caller+ gets its ConnectionError within a second timeout, not never."""++ class Host:+ async def new_stream(self, peer_id, protocols):+ await trio.sleep_forever()++ async def disconnect(self, peer_id):+ await trio.sleep_forever()++ async def main():+ started = trio.current_time()+ with pytest.raises(ConnectionError, match="within 10s"):+ await open_stream(Host(), ID.from_base58(ROUTER), ECHO, 10)+ assert trio.current_time() - started == pytest.approx(20)++ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))++def test_wss_is_dialed_with_certificate_verification():"""py-libp2p dials wss with verification off unless given a TLS clientcontext (libp2p/py-libp2p#1550); the host gives it the system roots, so