Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
sdk/python: return yamux backlog slots on close, bound every stream open
Antonio Ojeaa3ee48741a5a
diff --git a/sdk/README.md b/sdk/README.mdindex 1ae27069..6f2d9cb7 100644--- a/sdk/README.md+++ b/sdk/README.md@@ -90,6 +90,17 @@ pinned by a test:py-libp2p 0.7 cannot take this over: its relay client has the framingproblem above, and its `RelayDiscovery` reserves again only after theexpiry has passed, when the router has already dropped the slot.+- py-libp2p 0.7's yamux takes one of 256 per-connection backlog slots for+ every outbound stream and returns it only when sending the SYN fails, so+ the 257th `open_stream` on a connection blocks forever with no error. A+ member that keeps one connection to its router reaches that in hours (a+ DHT provide every ten minutes, a reservation renewal every hour). `main`+ releases the slot on close (libp2p/py-libp2p#1426); until that is+ released, `host.py` replaces `Yamux.open_stream` with one that does the+ same, since py-libp2p constructs `Yamux` by name whatever `muxer_opt`+ says. Every stream the SDK opens also goes through `open_stream`, which+ bounds `new_stream` with the caller's deadline, so a muxer that cannot+ open a stream fails the call instead of parking it.- py-libp2p's Kademlia client takes a protocol prefix but its providerlookups still speak `/ipfs/kad/1.0.0`, so it cannot reach the mesh DHT.The Python SDK does a bounded GET_PROVIDERS walk itself ondiff --git a/sdk/python/src/agent_mesh/auth.py b/sdk/python/src/agent_mesh/auth.pyindex 50111128..e9a1c90c 100644--- a/sdk/python/src/agent_mesh/auth.py+++ b/sdk/python/src/agent_mesh/auth.py@@ -28,6 +28,7 @@ from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytfrom ._proto import sam_pb2 as pbfrom .biscuit import BiscuitVerificationError, VerifiedBiscuit, verify_peer_biscuit+from .host import open_streamlogger = logging.getLogger("agent_mesh")@@ -57,7 +58,7 @@ async def _read_frame(stream: INetStream) -> bytes:async def authenticate_with_peer(host: IHost, peer_id: ID, frame: bytes, trusted_keys: Sequence[bytes]) -> VerifiedBiscuit:"""Client side: presents `frame` on a new /sam/auth/1.0.0 stream to aconnected peer and returns the peer's verified credential."""- stream = await host.new_stream(peer_id, [AUTH_PROTOCOL])+ stream = await open_stream(host, peer_id, AUTH_PROTOCOL, AUTH_HANDSHAKE_TIMEOUT)try:with trio.fail_after(AUTH_HANDSHAKE_TIMEOUT):await stream.write(encode_varint_prefixed(frame))diff --git a/sdk/python/src/agent_mesh/discovery.py b/sdk/python/src/agent_mesh/discovery.pyindex 302d752e..fa39c87b 100644--- a/sdk/python/src/agent_mesh/discovery.py+++ b/sdk/python/src/agent_mesh/discovery.py@@ -34,6 +34,8 @@ from libp2p.peer.id import IDfrom libp2p.peer.peerinfo import PeerInfofrom libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes+from .host import open_stream+logger = logging.getLogger("agent_mesh")# go-libp2p-kad-dht with dht.ProtocolPrefix("/sam").@@ -75,7 +77,7 @@ def parse_service_target(target: str) -> tuple[str, str]:async def _get_providers(host: IHost, peer_id: ID, key: bytes) -> kad.Message | None:try:- stream = await host.new_stream(peer_id, [DHT_PROTOCOL])+ stream = await open_stream(host, peer_id, DHT_PROTOCOL, _QUERY_TIMEOUT)except Exception as err: # noqa: BLE001 - a peer that does not serve the DHT is skippedlogger.debug("dht: %s does not answer %s: %s", peer_id, DHT_PROTOCOL, err)return None@@ -148,7 +150,7 @@ async def _add_provider(host: IHost, peer_id: ID, key: bytes, addrs: Sequence[mustores the record only when the sender is the provider and lists atleast one address, and answers nothing."""try:- stream = await host.new_stream(peer_id, [DHT_PROTOCOL])+ stream = await open_stream(host, peer_id, DHT_PROTOCOL, _QUERY_TIMEOUT)except Exception as err: # noqa: BLE001 - a peer that does not serve the DHT is skippedlogger.debug("dht: %s does not answer %s: %s", peer_id, DHT_PROTOCOL, err)return Falsediff --git a/sdk/python/src/agent_mesh/host.py b/sdk/python/src/agent_mesh/host.pyindex 69cdd3aa..54d5e721 100644--- a/sdk/python/src/agent_mesh/host.py+++ b/sdk/python/src/agent_mesh/host.py@@ -18,6 +18,7 @@ muxer. py-libp2p is trio-based, so everything here is trio async."""from __future__ import annotations+import importlib.metadataimport osfrom datetime import datetime, timedelta, timezonefrom typing import Sequence@@ -27,19 +28,73 @@ import triofrom cryptography import x509from cryptography.x509.oid import NameOIDfrom libp2p import new_host-from libp2p.abc import IHost+from libp2p.abc import IHost, INetStreamfrom libp2p.crypto.ed25519 import create_new_key_pairfrom libp2p.custom_types import TProtocol+from libp2p.peer.id import IDfrom libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addrfrom libp2p.security.tls.transport import PROTOCOL_ID as TLS_PROTOCOL_IDfrom libp2p.security.tls.transport import IdentityConfig, TLSTransportfrom libp2p.stream_muxer.yamux.yamux import PROTOCOL_ID as YAMUX_PROTOCOL_ID-from libp2p.stream_muxer.yamux.yamux import Yamux+from libp2p.stream_muxer.yamux.yamux import Yamux, YamuxStreamfrom multiaddr.resolvers import DNSResolverfrom .identity import Identity+def _libp2p_version() -> tuple[int, int]:+ major, minor = importlib.metadata.version("libp2p").split(".")[:2]+ return int(major), int(minor)+++# py-libp2p 0.7 takes one of the 256 slots of a connection's+# stream_backlog_semaphore for every outbound stream and gives it back only+# when sending the SYN fails, so after 256 streams on one connection every+# open_stream blocks forever, silently. main releases the slot when the local+# side closes the stream (libp2p/py-libp2p#1426); until that is released,+# Yamux.open_stream is replaced here to do the same. A subclass would not do:+# MuxerMultistream.new_conn constructs Yamux by name, whatever muxer_opt says.+# Delete this block and its test with the libp2p>=0.8 bump.+LIBP2P_LEAKS_STREAM_SLOTS = _libp2p_version() < (0, 8)+++def _release_slot_on_close(open_stream): # type: ignore[no-untyped-def]+ async def open_stream_releasing_slot(self: Yamux) -> YamuxStream:+ stream = await open_stream(self)+ release = self.stream_backlog_semaphore.release+ close, reset = stream.close, stream.reset+ released = False++ def release_once() -> None:+ nonlocal released+ if not released:+ released = True+ release()++ async def close_and_release() -> None:+ try:+ await close()+ finally:+ release_once()++ async def reset_and_release() -> None:+ try:+ await reset()+ finally:+ release_once()++ stream.close = close_and_release # type: ignore[method-assign]+ stream.reset = reset_and_release # type: ignore[method-assign]+ return stream++ open_stream_releasing_slot.releases_slot = True # type: ignore[attr-defined]+ return open_stream_releasing_slot+++if LIBP2P_LEAKS_STREAM_SLOTS and not getattr(Yamux.open_stream, "releases_slot", False):+ Yamux.open_stream = _release_slot_on_close(Yamux.open_stream) # type: ignore[method-assign]++def _certificate_template() -> x509.CertificateBuilder:"""The libp2p TLS certificate with nothing but the libp2p extension.py-libp2p's default template also adds BasicConstraints and KeyUsage,@@ -76,6 +131,17 @@ def create_mesh_host(identity: Identity, listen_addrs: Sequence[str] = ()) -> tureturn host, [multiaddr.Multiaddr(a) for a in listen_addrs]+async def open_stream(host: IHost, peer_id: ID, protocol: TProtocol, timeout: float) -> INetStream:+ """host.new_stream bounded by a timeout. py-libp2p bounds the protocol+ negotiation but not the muxer, and a muxer that cannot open a stream+ would otherwise park the caller forever without a word."""+ try:+ with trio.fail_after(timeout):+ return await host.new_stream(peer_id, [protocol])+ except trio.TooSlowError:+ raise ConnectionError(f"no {protocol} stream to {peer_id} within {timeout:g}s") from None++_DNS_PROTOCOLS = frozenset({"dnsaddr", "dns", "dns4", "dns6"})# py-libp2p dials TCP only; a resolved address on another transport is noise._UNDIALABLE_PROTOCOLS = frozenset({"quic", "quic-v1", "ws", "wss", "webtransport", "webrtc", "webrtc-direct"})diff --git a/sdk/python/src/agent_mesh/mcp_client.py b/sdk/python/src/agent_mesh/mcp_client.pyindex b432a32e..8cebb624 100644--- a/sdk/python/src/agent_mesh/mcp_client.py+++ b/sdk/python/src/agent_mesh/mcp_client.py@@ -35,6 +35,7 @@ from mcp.shared.message import SessionMessagefrom ._proto import sam_pb2 as pbfrom .auth import AUTH_HANDSHAKE_TIMEOUT, MAX_AUTH_FRAME_BYTES, MCP_PROTOCOL, AuthRejectedErrorfrom .biscuit import BiscuitVerificationError, VerifiedBiscuit, verify_peer_biscuit+from .host import open_streamlogger = logging.getLogger("agent_mesh")@@ -96,7 +97,7 @@ async def open_mcp_session("""Opens /sam/mcp/1.0.0 to a connected provider with `frame`, this member'sAuthFrame naming the service, verifies the provider and yields aninitialized MCP ClientSession with the provider's credential."""- stream = await host.new_stream(peer_id, [MCP_PROTOCOL])+ stream = await open_stream(host, peer_id, MCP_PROTOCOL, AUTH_HANDSHAKE_TIMEOUT)try:with trio.fail_after(AUTH_HANDSHAKE_TIMEOUT):await stream.write(encode_varint_prefixed(frame))diff --git a/sdk/python/src/agent_mesh/relay.py b/sdk/python/src/agent_mesh/relay.pyindex 9896027e..17704b6c 100644--- a/sdk/python/src/agent_mesh/relay.py+++ b/sdk/python/src/agent_mesh/relay.py@@ -34,6 +34,7 @@ from libp2p.peer.id import IDfrom libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytesfrom ._proto import circuit_pb2 as circuit+from .host import open_streamfrom .identity import canonical_peer_idlogger = logging.getLogger("agent_mesh")@@ -60,7 +61,7 @@ async def reserve_relay(host: IHost, relay_peer_id: ID) -> circuit.Reservation:"""Asks a connected relay for a reservation and returns it. A router grantsone only to a peer that passed the auth handshake, so success is also proofof admission."""- stream = await host.new_stream(relay_peer_id, [HOP_PROTOCOL])+ stream = await open_stream(host, relay_peer_id, HOP_PROTOCOL, RELAY_MESSAGE_TIMEOUT)try:with trio.fail_after(RELAY_MESSAGE_TIMEOUT):req = circuit.HopMessage(type=circuit.HopMessage.RESERVE)@@ -80,7 +81,7 @@ async def reserve_relay(host: IHost, relay_peer_id: ID) -> circuit.Reservation:async def dial_through_relay(host: IHost, relay_peer_id: ID, target: ID) -> INetConn:"""Opens a connection to `target` through a relay we are connected to, andregisters it with the host so streams can be opened on it."""- stream = await host.new_stream(relay_peer_id, [HOP_PROTOCOL])+ stream = await open_stream(host, relay_peer_id, HOP_PROTOCOL, RELAY_MESSAGE_TIMEOUT)try:with trio.fail_after(RELAY_MESSAGE_TIMEOUT):req = circuit.HopMessage(type=circuit.HopMessage.CONNECT, peer=circuit.Peer(id=target.to_bytes()))diff --git a/sdk/python/src/agent_mesh/serve.py b/sdk/python/src/agent_mesh/serve.pyindex cfa8e499..691d2f41 100644--- a/sdk/python/src/agent_mesh/serve.py+++ b/sdk/python/src/agent_mesh/serve.py@@ -41,6 +41,7 @@ from .auth import AUTH_HANDSHAKE_TIMEOUT, MAX_AUTH_FRAME_BYTESfrom .authorizer import AuthorizationError, AuthorizeRequest, ProviderAuthorizerOptions, authorize_callerfrom .biscuit import VerifiedBiscuitfrom .discovery import ServiceType+from .host import open_streamfrom .mcp_client import MAX_MCP_MESSAGE_BYTESlogger = logging.getLogger("agent_mesh")@@ -450,7 +451,7 @@ async def http_request_over_stream(out.append(("content-length", str(len(payload))))conn = h11.Connection(h11.CLIENT)- stream = await host.new_stream(peer_id, [HTTP_PROTOCOL])+ stream = await open_stream(host, peer_id, HTTP_PROTOCOL, timeout)try:with trio.fail_after(timeout):data = conn.send(h11.Request(method=method, target=f"/{scheme}/{name}" + (path if path.startswith("/") else "/" + path), headers=out)) or b""diff --git a/sdk/python/tests/test_host.py b/sdk/python/tests/test_host.pyindex 9e6996cd..ac55f66c 100644--- a/sdk/python/tests/test_host.py+++ b/sdk/python/tests/test_host.py@@ -14,17 +14,21 @@"""What a dial must do that py-libp2p does not: resolve a /dnsaddr routeraddress through its TXT records and keep the transports this host has-(dial_addrs), and give up on an address nobody answers (dial)."""+(dial_addrs), and give up on an address nobody answers (dial). And what a+stream must do: come back after the 256th one on a connection (py-libp2p+0.7 leaks its yamux backlog slots), or fail at a deadline."""import multiaddrimport pytestimport trioimport trio.testing+from libp2p.custom_types import TProtocolfrom libp2p.peer.id import ID-from libp2p.peer.peerinfo import PeerInfo+from libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addrfrom multiaddr.resolvers import DNSResolver-from agent_mesh.host import dial_addrs, peer_info+from agent_mesh.host import LIBP2P_LEAKS_STREAM_SLOTS, create_mesh_host, dial_addrs, open_stream, peer_info+from agent_mesh.identity import Identityfrom agent_mesh.session import DIAL_TIMEOUT, dialROUTER = "12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88"@@ -114,3 +118,60 @@ def test_a_dial_nobody_answers_ends_at_the_timeout():assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))+++ECHO = TProtocol("/test/echo/1.0.0")+++def test_a_connection_outlives_the_yamux_stream_backlog():+ """A member keeps one connection to its router for days and opens a+ short stream on it every few minutes (DHT provide, reservation renewal).+ py-libp2p 0.7 never returns a closed stream's slot to the connection's+ 256-slot backlog, so the 257th open_stream parks forever; ours must not."""++ async def echo(stream):+ try:+ await stream.write(await stream.read(64))+ finally:+ await stream.close()++ async def main():+ server, server_listen = create_mesh_host(Identity.generate(), ["/ip4/127.0.0.1/tcp/0"])+ client, _ = create_mesh_host(Identity.generate())+ server.set_stream_handler(ECHO, echo)+ async with server.run(listen_addrs=server_listen), client.run(listen_addrs=[]):+ await client.connect(info_from_p2p_addr(multiaddr.Multiaddr(f"{server.get_addrs()[0]}")))+ for i in range(300):+ # Alternate the two ways a caller lets go of a stream.+ with trio.fail_after(5):+ stream = await open_stream(client, server.get_id(), ECHO, 5)+ try:+ await stream.write(b"ping")+ assert await stream.read(4) == b"ping"+ finally:+ await (stream.close() if i % 2 else stream.reset())++ trio.run(main)+++def test_open_stream_ends_at_its_timeout():+ class Host:+ async def new_stream(self, peer_id, protocols):+ await trio.sleep_forever()++ async def main():+ started = trio.current_time()+ with pytest.raises(ConnectionError, match="within 10s"):+ await open_stream(Host(), ID.from_base58(ROUTER), ECHO, 10)+ assert trio.current_time() - started == pytest.approx(10)++ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))+++def test_the_stream_slot_workaround_retires_with_libp2p_0_8():+ """When the pin moves past 0.7, delete the Yamux.open_stream patch and+ this flag in host.py, and the backlog test above."""+ import importlib.metadata++ major, minor = (int(p) for p in importlib.metadata.version("libp2p").split(".")[:2])+ assert LIBP2P_LEAKS_STREAM_SLOTS == ((major, minor) < (0, 8))