Skip to content

Agent Mesh

sdk/python: bound every dial the DHT walk makes

Antonio Ojeaa18bc97d10b9
diff --git a/.github/k8s/sam-sdk-probe-cronjob-template.yaml b/.github/k8s/sam-sdk-probe-cronjob-template.yaml
index 0a84389a..83ddb5ec 100644
--- a/.github/k8s/sam-sdk-probe-cronjob-template.yaml
+++ b/.github/k8s/sam-sdk-probe-cronjob-template.yaml
@@ -236,11 +236,13 @@ data:
attempts=0
while :; do
attempts=$((attempts + 1))
- out=$($program "$@" 2>&1); rc=$?
+ # A program that hangs is a failure to report, not a job to let the deadline kill silently.
+ out=$(timeout -k 5 120 $program "$@" 2>&1); rc=$?
if [ $rc -eq 0 ] && printf '%s' "$out" | grep -q "$want"; then
PROVIDER=$(printf '%s' "$out" | grep -o 'is served by [^ ]*' | tail -1 | cut -d' ' -f4)
return 0
fi
+ [ $rc -eq 124 ] && out="$out (hung for 120s)"
[ $(( $(date +%s) - stage_start )) -ge 120 ] && fail "$stage" "no answer with '$want' in 120s ($attempts attempts; last: $(printf '%s' "$out" | tail -c 200))"
sleep 5
done
diff --git a/sdk/python/src/agent_mesh/discovery.py b/sdk/python/src/agent_mesh/discovery.py
index e887ae73..bcf7a66c 100644
--- a/sdk/python/src/agent_mesh/discovery.py
+++ b/sdk/python/src/agent_mesh/discovery.py
@@ -35,7 +35,7 @@ from libp2p.peer.id import ID
from libp2p.peer.peerinfo import PeerInfo
from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes
-from .host import open_stream
+from .host import dial, open_stream
logger = logging.getLogger("agent_mesh")
@@ -152,8 +152,9 @@ async def find_peer(host: IHost, target: ID, seeds: Iterable[ID]) -> list[multia
continue
if info.peer_id not in host.get_connected_peers():
try:
- await host.connect(info)
+ await dial(host, info)
except Exception: # noqa: BLE001 - unreachable closer peers are skipped
+ asked.add(info.peer_id)
continue
frontier.append(info.peer_id)
return []
@@ -190,8 +191,9 @@ async def find_providers(host: IHost, key: bytes, seeds: Iterable[ID], limit: in
continue
if info.peer_id not in host.get_connected_peers():
try:
- await host.connect(info)
+ await dial(host, info)
except Exception: # noqa: BLE001 - unreachable closer peers are skipped
+ asked.add(info.peer_id)
continue
frontier.append(info.peer_id)
if found:
diff --git a/sdk/python/src/agent_mesh/host.py b/sdk/python/src/agent_mesh/host.py
index 1576b375..1482575f 100644
--- a/sdk/python/src/agent_mesh/host.py
+++ b/sdk/python/src/agent_mesh/host.py
@@ -122,6 +122,24 @@ def _dial_websockets_by_name(transport: WebsocketTransport) -> None:
transport.dial = dial # type: ignore[method-assign]
+# sam-node's swarm dial timeout. py-libp2p has none of its own: a SYN to an
+# address nobody answers waits on the kernel, about two minutes, and is then
+# retried, and a provider record can name a pod a rollout just replaced. A
+# router the control plane lists may be unreachable from where a member runs
+# (a public address a network policy drops), and the routers' DHT names it as
+# a closer peer to every lookup.
+DIAL_TIMEOUT = 15.0
+
+
+async def dial(host: IHost, info: PeerInfo) -> None:
+ """host.connect, bounded by DIAL_TIMEOUT."""
+ try:
+ with trio.fail_after(DIAL_TIMEOUT):
+ await host.connect(info)
+ except trio.TooSlowError:
+ raise ConnectionError(f"no connection to {info.peer_id} within {DIAL_TIMEOUT:g}s") from None
+
+
async def open_stream(host: IHost, peer_id: ID, protocol: TProtocol, timeout: float) -> INetStream:
"""host.new_stream bounded by a timeout. py-libp2p bounds the protocol
negotiation but not the muxer, and a muxer that cannot open a stream
@@ -133,7 +151,9 @@ async def open_stream(host: IHost, peer_id: ID, protocol: TProtocol, timeout: fl
with trio.fail_after(timeout):
return await host.new_stream(peer_id, [protocol])
except trio.TooSlowError:
- with trio.CancelScope(shield=True):
+ # Closing writes a GO_AWAY on the very connection that stalled; give
+ # it a moment, then let the peer time the socket out on its own.
+ with trio.CancelScope(shield=True), trio.move_on_after(timeout):
try:
await host.disconnect(peer_id)
except Exception: # noqa: BLE001 - already gone
diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.py
index 2aa52c90..34c2fe54 100644
--- a/sdk/python/src/agent_mesh/session.py
+++ b/sdk/python/src/agent_mesh/session.py
@@ -41,7 +41,7 @@ from .auth import AUTH_PROTOCOL, auth_stream_handler, authenticate_with_peer
from .authorizer import ProviderAuthorizerOptions
from .biscuit import ROLE_ROUTER, VerifiedBiscuit, require_role
from .discovery import DiscoveredProvider, find_peer, find_providers, parse_service_target, service_key
-from .host import create_mesh_host, dial_addrs, peer_info
+from .host import create_mesh_host, dial, dial_addrs, peer_info
from .httpx_transport import MESH_PATH_PREFIX
from .identity import canonical_peer_id
from .libp2p_http import (
@@ -82,11 +82,6 @@ DEFAULT_CONTROL_PLANE_SYNC = 15 * 60.0
FIRST_CONTROL_PLANE_SYNC = 2.0
DEFAULT_CONTROL_PLANE_SYNC_JITTER = 2.0
-# sam-node's swarm dial timeout. py-libp2p has none of its own: a SYN to an
-# address nobody answers waits on the kernel, about two minutes, and is then
-# retried, and a provider record can name a pod a rollout just replaced.
-DIAL_TIMEOUT = 15.0
-
# How a caller names the peer it wants to reach: a provider `discover` returned,
# a peer id, or a multiaddr. For a provider or a peer id the SDK dials the
# addresses the peer advertised and then the relayed path through every router
@@ -100,15 +95,6 @@ def parse_peer_id(text: str) -> ID:
return ID.from_base58(canonical_peer_id(text))
-async def dial(host: IHost, info: PeerInfo) -> None:
- """host.connect, bounded by DIAL_TIMEOUT."""
- try:
- with trio.fail_after(DIAL_TIMEOUT):
- await host.connect(info)
- except trio.TooSlowError:
- raise ConnectionError(f"no connection to {info.peer_id} within {DIAL_TIMEOUT:g}s") from None
-
-
def canonical_peer_ids(ids: Sequence[str]) -> list[str]:
"""Canonicalizes a list from the control plane, dropping entries that are
not peer IDs: they can match nothing, so they ban nothing."""
diff --git a/sdk/python/tests/test_discovery.py b/sdk/python/tests/test_discovery.py
new file mode 100644
index 00000000..6a779a49
--- /dev/null
+++ b/sdk/python/tests/test_discovery.py
@@ -0,0 +1,147 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The DHT walks against a fake host: what they do with the closer peers a
+router names. The wire format against a real router is covered by
+tests/integration/sdk_mesh_test.go."""
+
+import multiaddr
+import pytest
+import trio
+import trio.testing
+from libp2p.kad_dht.pb import kademlia_pb2 as kad
+from libp2p.peer.id import ID
+from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes
+
+from agent_mesh.discovery import DHT_PROTOCOL, find_peer, find_providers
+from agent_mesh.host import DIAL_TIMEOUT
+
+SELF = ID.from_base58("12D3KooWJoJPhXqLrGAcVsGSzHMzWrmoHCWoTzVSSiT3JfR7RBVo")
+ROUTER = ID.from_base58("12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88")
+DARK_ROUTER = ID.from_base58("12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM")
+TARGET = ID.from_base58("12D3KooWQYhTNQdmr3ArTeUHRYzFg94BKyTkoWBDWez9kSCVe2Xo")
+
+
+class _Stream:
+ """One kad request answered from a table, as a router's DHT would."""
+
+ def __init__(self, answer):
+ self._answer = answer
+ self._req = b""
+ self._resp = b""
+
+ async def write(self, data):
+ self._req += data
+
+ async def read(self, n=None):
+ if self._req:
+ req = kad.Message.FromString(await read_varint_prefixed_bytes(_Bytes(self._req)))
+ self._req = b""
+ self._resp = encode_varint_prefixed(self._answer(req).SerializeToString())
+ n = len(self._resp) if n is None else n
+ out, self._resp = self._resp[:n], self._resp[n:]
+ return out
+
+ async def close(self):
+ pass
+
+
+class _Bytes:
+ def __init__(self, data):
+ self._data = data
+
+ async def read(self, n=None):
+ n = len(self._data) if n is None else n
+ out, self._data = self._data[:n], self._data[n:]
+ return out
+
+
+class _Host:
+ """Connected to one router; the other one the router names never
+ answers a SYN."""
+
+ def __init__(self, answers):
+ self._answers = answers
+ self.connected = {ROUTER}
+ self.dials = []
+
+ def get_id(self):
+ return SELF
+
+ def get_connected_peers(self):
+ return list(self.connected)
+
+ async def connect(self, info):
+ self.dials.append(info.peer_id)
+ await trio.sleep_forever()
+
+ async def new_stream(self, peer_id, protocols):
+ assert protocols == [DHT_PROTOCOL]
+ assert peer_id in self.connected
+ return _Stream(self._answers[peer_id])
+
+ async def disconnect(self, peer_id):
+ self.connected.discard(peer_id)
+
+
+def _peer(peer_id, *addrs):
+ return kad.Message.Peer(id=peer_id.to_bytes(), addrs=[multiaddr.Multiaddr(a).to_bytes() for a in addrs])
+
+
+def _names_the_dark_router(req):
+ return kad.Message(type=req.type, closerPeers=[_peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501")])
+
+
+def test_find_providers_dials_an_unanswering_closer_peer_once():
+ """A router the control plane lists may be dark from where a member runs
+ (a public address a network policy drops). Every router names it as
+ closer to every key; the walk dials it once, bounded, then moves on."""
+ host = _Host({ROUTER: _names_the_dark_router})
+
+ async def main():
+ started = trio.current_time()
+ assert await find_providers(host, b"key", [ROUTER]) == []
+ assert host.dials == [DARK_ROUTER]
+ assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)
+
+ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+
+
+def test_find_peer_dials_an_unanswering_closer_peer_once():
+ host = _Host({ROUTER: _names_the_dark_router})
+
+ async def main():
+ started = trio.current_time()
+ assert await find_peer(host, TARGET, [ROUTER]) == []
+ assert host.dials == [DARK_ROUTER]
+ assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)
+
+ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+
+
+def test_find_peer_returns_the_target_a_router_knows():
+ def knows_target(req):
+ return kad.Message(
+ type=req.type,
+ closerPeers=[_peer(TARGET, "/ip4/10.0.0.9/tcp/4001"), _peer(DARK_ROUTER, "/ip4/203.0.113.7/tcp/4501")],
+ )
+
+ host = _Host({ROUTER: knows_target})
+
+ async def main():
+ addrs = await find_peer(host, TARGET, [ROUTER])
+ assert [str(a) for a in addrs] == ["/ip4/10.0.0.9/tcp/4001"]
+ assert host.dials == []
+
+ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
diff --git a/sdk/python/tests/test_host.py b/sdk/python/tests/test_host.py
index 1770a96c..7da7360b 100644
--- a/sdk/python/tests/test_host.py
+++ b/sdk/python/tests/test_host.py
@@ -32,9 +32,8 @@ from libp2p.stream_muxer.yamux.yamux import FLAG_SYN, YAMUX_HEADER_FORMAT
from libp2p.transport.websocket.transport import WebsocketTransport
from multiaddr.resolvers import DNSResolver
-from agent_mesh.host import create_mesh_host, dial_addrs, open_stream, peer_info
+from agent_mesh.host import DIAL_TIMEOUT, create_mesh_host, dial, dial_addrs, open_stream, peer_info
from agent_mesh.identity import Identity
-from agent_mesh.session import DIAL_TIMEOUT, dial
ROUTER = "12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88"
OTHER = "12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM"
@@ -274,6 +273,27 @@ def test_open_stream_ends_at_its_timeout():
trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+def test_hanging_up_on_a_stalled_connection_is_bounded_too():
+ """Closing the connection writes a GO_AWAY on the very connection whose
+ SYN stalled; on a blackholed peer that write parks as well. The caller
+ gets its ConnectionError within a second timeout, not never."""
+
+ class Host:
+ async def new_stream(self, peer_id, protocols):
+ await trio.sleep_forever()
+
+ async def disconnect(self, peer_id):
+ await trio.sleep_forever()
+
+ async def main():
+ started = trio.current_time()
+ with pytest.raises(ConnectionError, match="within 10s"):
+ await open_stream(Host(), ID.from_base58(ROUTER), ECHO, 10)
+ assert trio.current_time() - started == pytest.approx(20)
+
+ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+
+
def test_wss_is_dialed_with_certificate_verification():
"""py-libp2p dials wss with verification off unless given a TLS client
context (libp2p/py-libp2p#1550); the host gives it the system roots, so