Skip to content

Agent Mesh

sdk/python: return yamux backlog slots on close, bound every stream open

Antonio Ojeaa3ee48741a5a
diff --git a/sdk/README.md b/sdk/README.md
index 1ae27069..6f2d9cb7 100644
--- a/sdk/README.md
+++ b/sdk/README.md
@@ -90,6 +90,17 @@ pinned by a test:
py-libp2p 0.7 cannot take this over: its relay client has the framing
problem above, and its `RelayDiscovery` reserves again only after the
expiry has passed, when the router has already dropped the slot.
+- py-libp2p 0.7's yamux takes one of 256 per-connection backlog slots for
+ every outbound stream and returns it only when sending the SYN fails, so
+ the 257th `open_stream` on a connection blocks forever with no error. A
+ member that keeps one connection to its router reaches that in hours (a
+ DHT provide every ten minutes, a reservation renewal every hour). `main`
+ releases the slot on close (libp2p/py-libp2p#1426); until that is
+ released, `host.py` replaces `Yamux.open_stream` with one that does the
+ same, since py-libp2p constructs `Yamux` by name whatever `muxer_opt`
+ says. Every stream the SDK opens also goes through `open_stream`, which
+ bounds `new_stream` with the caller's deadline, so a muxer that cannot
+ open a stream fails the call instead of parking it.
- py-libp2p's Kademlia client takes a protocol prefix but its provider
lookups still speak `/ipfs/kad/1.0.0`, so it cannot reach the mesh DHT.
The Python SDK does a bounded GET_PROVIDERS walk itself on
diff --git a/sdk/python/src/agent_mesh/auth.py b/sdk/python/src/agent_mesh/auth.py
index 50111128..e9a1c90c 100644
--- a/sdk/python/src/agent_mesh/auth.py
+++ b/sdk/python/src/agent_mesh/auth.py
@@ -28,6 +28,7 @@ from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_byt
from ._proto import sam_pb2 as pb
from .biscuit import BiscuitVerificationError, VerifiedBiscuit, verify_peer_biscuit
+from .host import open_stream
logger = logging.getLogger("agent_mesh")
@@ -57,7 +58,7 @@ async def _read_frame(stream: INetStream) -> bytes:
async def authenticate_with_peer(host: IHost, peer_id: ID, frame: bytes, trusted_keys: Sequence[bytes]) -> VerifiedBiscuit:
"""Client side: presents `frame` on a new /sam/auth/1.0.0 stream to a
connected peer and returns the peer's verified credential."""
- stream = await host.new_stream(peer_id, [AUTH_PROTOCOL])
+ stream = await open_stream(host, peer_id, AUTH_PROTOCOL, AUTH_HANDSHAKE_TIMEOUT)
try:
with trio.fail_after(AUTH_HANDSHAKE_TIMEOUT):
await stream.write(encode_varint_prefixed(frame))
diff --git a/sdk/python/src/agent_mesh/discovery.py b/sdk/python/src/agent_mesh/discovery.py
index 302d752e..fa39c87b 100644
--- a/sdk/python/src/agent_mesh/discovery.py
+++ b/sdk/python/src/agent_mesh/discovery.py
@@ -34,6 +34,8 @@ from libp2p.peer.id import ID
from libp2p.peer.peerinfo import PeerInfo
from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes
+from .host import open_stream
+
logger = logging.getLogger("agent_mesh")
# go-libp2p-kad-dht with dht.ProtocolPrefix("/sam").
@@ -75,7 +77,7 @@ def parse_service_target(target: str) -> tuple[str, str]:
async def _get_providers(host: IHost, peer_id: ID, key: bytes) -> kad.Message | None:
try:
- stream = await host.new_stream(peer_id, [DHT_PROTOCOL])
+ stream = await open_stream(host, peer_id, DHT_PROTOCOL, _QUERY_TIMEOUT)
except Exception as err: # noqa: BLE001 - a peer that does not serve the DHT is skipped
logger.debug("dht: %s does not answer %s: %s", peer_id, DHT_PROTOCOL, err)
return None
@@ -148,7 +150,7 @@ async def _add_provider(host: IHost, peer_id: ID, key: bytes, addrs: Sequence[mu
stores the record only when the sender is the provider and lists at
least one address, and answers nothing."""
try:
- stream = await host.new_stream(peer_id, [DHT_PROTOCOL])
+ stream = await open_stream(host, peer_id, DHT_PROTOCOL, _QUERY_TIMEOUT)
except Exception as err: # noqa: BLE001 - a peer that does not serve the DHT is skipped
logger.debug("dht: %s does not answer %s: %s", peer_id, DHT_PROTOCOL, err)
return False
diff --git a/sdk/python/src/agent_mesh/host.py b/sdk/python/src/agent_mesh/host.py
index 69cdd3aa..54d5e721 100644
--- a/sdk/python/src/agent_mesh/host.py
+++ b/sdk/python/src/agent_mesh/host.py
@@ -18,6 +18,7 @@ muxer. py-libp2p is trio-based, so everything here is trio async."""
from __future__ import annotations
+import importlib.metadata
import os
from datetime import datetime, timedelta, timezone
from typing import Sequence
@@ -27,19 +28,73 @@ import trio
from cryptography import x509
from cryptography.x509.oid import NameOID
from libp2p import new_host
-from libp2p.abc import IHost
+from libp2p.abc import IHost, INetStream
from libp2p.crypto.ed25519 import create_new_key_pair
from libp2p.custom_types import TProtocol
+from libp2p.peer.id import ID
from libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addr
from libp2p.security.tls.transport import PROTOCOL_ID as TLS_PROTOCOL_ID
from libp2p.security.tls.transport import IdentityConfig, TLSTransport
from libp2p.stream_muxer.yamux.yamux import PROTOCOL_ID as YAMUX_PROTOCOL_ID
-from libp2p.stream_muxer.yamux.yamux import Yamux
+from libp2p.stream_muxer.yamux.yamux import Yamux, YamuxStream
from multiaddr.resolvers import DNSResolver
from .identity import Identity
+def _libp2p_version() -> tuple[int, int]:
+ major, minor = importlib.metadata.version("libp2p").split(".")[:2]
+ return int(major), int(minor)
+
+
+# py-libp2p 0.7 takes one of the 256 slots of a connection's
+# stream_backlog_semaphore for every outbound stream and gives it back only
+# when sending the SYN fails, so after 256 streams on one connection every
+# open_stream blocks forever, silently. main releases the slot when the local
+# side closes the stream (libp2p/py-libp2p#1426); until that is released,
+# Yamux.open_stream is replaced here to do the same. A subclass would not do:
+# MuxerMultistream.new_conn constructs Yamux by name, whatever muxer_opt says.
+# Delete this block and its test with the libp2p>=0.8 bump.
+LIBP2P_LEAKS_STREAM_SLOTS = _libp2p_version() < (0, 8)
+
+
+def _release_slot_on_close(open_stream): # type: ignore[no-untyped-def]
+ async def open_stream_releasing_slot(self: Yamux) -> YamuxStream:
+ stream = await open_stream(self)
+ release = self.stream_backlog_semaphore.release
+ close, reset = stream.close, stream.reset
+ released = False
+
+ def release_once() -> None:
+ nonlocal released
+ if not released:
+ released = True
+ release()
+
+ async def close_and_release() -> None:
+ try:
+ await close()
+ finally:
+ release_once()
+
+ async def reset_and_release() -> None:
+ try:
+ await reset()
+ finally:
+ release_once()
+
+ stream.close = close_and_release # type: ignore[method-assign]
+ stream.reset = reset_and_release # type: ignore[method-assign]
+ return stream
+
+ open_stream_releasing_slot.releases_slot = True # type: ignore[attr-defined]
+ return open_stream_releasing_slot
+
+
+if LIBP2P_LEAKS_STREAM_SLOTS and not getattr(Yamux.open_stream, "releases_slot", False):
+ Yamux.open_stream = _release_slot_on_close(Yamux.open_stream) # type: ignore[method-assign]
+
+
def _certificate_template() -> x509.CertificateBuilder:
"""The libp2p TLS certificate with nothing but the libp2p extension.
py-libp2p's default template also adds BasicConstraints and KeyUsage,
@@ -76,6 +131,17 @@ def create_mesh_host(identity: Identity, listen_addrs: Sequence[str] = ()) -> tu
return host, [multiaddr.Multiaddr(a) for a in listen_addrs]
+async def open_stream(host: IHost, peer_id: ID, protocol: TProtocol, timeout: float) -> INetStream:
+ """host.new_stream bounded by a timeout. py-libp2p bounds the protocol
+ negotiation but not the muxer, and a muxer that cannot open a stream
+ would otherwise park the caller forever without a word."""
+ try:
+ with trio.fail_after(timeout):
+ return await host.new_stream(peer_id, [protocol])
+ except trio.TooSlowError:
+ raise ConnectionError(f"no {protocol} stream to {peer_id} within {timeout:g}s") from None
+
+
_DNS_PROTOCOLS = frozenset({"dnsaddr", "dns", "dns4", "dns6"})
# py-libp2p dials TCP only; a resolved address on another transport is noise.
_UNDIALABLE_PROTOCOLS = frozenset({"quic", "quic-v1", "ws", "wss", "webtransport", "webrtc", "webrtc-direct"})
diff --git a/sdk/python/src/agent_mesh/mcp_client.py b/sdk/python/src/agent_mesh/mcp_client.py
index b432a32e..8cebb624 100644
--- a/sdk/python/src/agent_mesh/mcp_client.py
+++ b/sdk/python/src/agent_mesh/mcp_client.py
@@ -35,6 +35,7 @@ from mcp.shared.message import SessionMessage
from ._proto import sam_pb2 as pb
from .auth import AUTH_HANDSHAKE_TIMEOUT, MAX_AUTH_FRAME_BYTES, MCP_PROTOCOL, AuthRejectedError
from .biscuit import BiscuitVerificationError, VerifiedBiscuit, verify_peer_biscuit
+from .host import open_stream
logger = logging.getLogger("agent_mesh")
@@ -96,7 +97,7 @@ async def open_mcp_session(
"""Opens /sam/mcp/1.0.0 to a connected provider with `frame`, this member's
AuthFrame naming the service, verifies the provider and yields an
initialized MCP ClientSession with the provider's credential."""
- stream = await host.new_stream(peer_id, [MCP_PROTOCOL])
+ stream = await open_stream(host, peer_id, MCP_PROTOCOL, AUTH_HANDSHAKE_TIMEOUT)
try:
with trio.fail_after(AUTH_HANDSHAKE_TIMEOUT):
await stream.write(encode_varint_prefixed(frame))
diff --git a/sdk/python/src/agent_mesh/relay.py b/sdk/python/src/agent_mesh/relay.py
index 9896027e..17704b6c 100644
--- a/sdk/python/src/agent_mesh/relay.py
+++ b/sdk/python/src/agent_mesh/relay.py
@@ -34,6 +34,7 @@ from libp2p.peer.id import ID
from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes
from ._proto import circuit_pb2 as circuit
+from .host import open_stream
from .identity import canonical_peer_id
logger = logging.getLogger("agent_mesh")
@@ -60,7 +61,7 @@ async def reserve_relay(host: IHost, relay_peer_id: ID) -> circuit.Reservation:
"""Asks a connected relay for a reservation and returns it. A router grants
one only to a peer that passed the auth handshake, so success is also proof
of admission."""
- stream = await host.new_stream(relay_peer_id, [HOP_PROTOCOL])
+ stream = await open_stream(host, relay_peer_id, HOP_PROTOCOL, RELAY_MESSAGE_TIMEOUT)
try:
with trio.fail_after(RELAY_MESSAGE_TIMEOUT):
req = circuit.HopMessage(type=circuit.HopMessage.RESERVE)
@@ -80,7 +81,7 @@ async def reserve_relay(host: IHost, relay_peer_id: ID) -> circuit.Reservation:
async def dial_through_relay(host: IHost, relay_peer_id: ID, target: ID) -> INetConn:
"""Opens a connection to `target` through a relay we are connected to, and
registers it with the host so streams can be opened on it."""
- stream = await host.new_stream(relay_peer_id, [HOP_PROTOCOL])
+ stream = await open_stream(host, relay_peer_id, HOP_PROTOCOL, RELAY_MESSAGE_TIMEOUT)
try:
with trio.fail_after(RELAY_MESSAGE_TIMEOUT):
req = circuit.HopMessage(type=circuit.HopMessage.CONNECT, peer=circuit.Peer(id=target.to_bytes()))
diff --git a/sdk/python/src/agent_mesh/serve.py b/sdk/python/src/agent_mesh/serve.py
index cfa8e499..691d2f41 100644
--- a/sdk/python/src/agent_mesh/serve.py
+++ b/sdk/python/src/agent_mesh/serve.py
@@ -41,6 +41,7 @@ from .auth import AUTH_HANDSHAKE_TIMEOUT, MAX_AUTH_FRAME_BYTES
from .authorizer import AuthorizationError, AuthorizeRequest, ProviderAuthorizerOptions, authorize_caller
from .biscuit import VerifiedBiscuit
from .discovery import ServiceType
+from .host import open_stream
from .mcp_client import MAX_MCP_MESSAGE_BYTES
logger = logging.getLogger("agent_mesh")
@@ -450,7 +451,7 @@ async def http_request_over_stream(
out.append(("content-length", str(len(payload))))
conn = h11.Connection(h11.CLIENT)
- stream = await host.new_stream(peer_id, [HTTP_PROTOCOL])
+ stream = await open_stream(host, peer_id, HTTP_PROTOCOL, timeout)
try:
with trio.fail_after(timeout):
data = conn.send(h11.Request(method=method, target=f"/{scheme}/{name}" + (path if path.startswith("/") else "/" + path), headers=out)) or b""
diff --git a/sdk/python/tests/test_host.py b/sdk/python/tests/test_host.py
index 9e6996cd..ac55f66c 100644
--- a/sdk/python/tests/test_host.py
+++ b/sdk/python/tests/test_host.py
@@ -14,17 +14,21 @@
"""What a dial must do that py-libp2p does not: resolve a /dnsaddr router
address through its TXT records and keep the transports this host has
-(dial_addrs), and give up on an address nobody answers (dial)."""
+(dial_addrs), and give up on an address nobody answers (dial). And what a
+stream must do: come back after the 256th one on a connection (py-libp2p
+0.7 leaks its yamux backlog slots), or fail at a deadline."""
import multiaddr
import pytest
import trio
import trio.testing
+from libp2p.custom_types import TProtocol
from libp2p.peer.id import ID
-from libp2p.peer.peerinfo import PeerInfo
+from libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addr
from multiaddr.resolvers import DNSResolver
-from agent_mesh.host import dial_addrs, peer_info
+from agent_mesh.host import LIBP2P_LEAKS_STREAM_SLOTS, create_mesh_host, dial_addrs, open_stream, peer_info
+from agent_mesh.identity import Identity
from agent_mesh.session import DIAL_TIMEOUT, dial
ROUTER = "12D3KooWG1pA6goegCncqwbZLSr8pnjUZ6JMAAe6SmnHTgUNCk88"
@@ -114,3 +118,60 @@ def test_a_dial_nobody_answers_ends_at_the_timeout():
assert trio.current_time() - started == pytest.approx(DIAL_TIMEOUT)
trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+
+
+ECHO = TProtocol("/test/echo/1.0.0")
+
+
+def test_a_connection_outlives_the_yamux_stream_backlog():
+ """A member keeps one connection to its router for days and opens a
+ short stream on it every few minutes (DHT provide, reservation renewal).
+ py-libp2p 0.7 never returns a closed stream's slot to the connection's
+ 256-slot backlog, so the 257th open_stream parks forever; ours must not."""
+
+ async def echo(stream):
+ try:
+ await stream.write(await stream.read(64))
+ finally:
+ await stream.close()
+
+ async def main():
+ server, server_listen = create_mesh_host(Identity.generate(), ["/ip4/127.0.0.1/tcp/0"])
+ client, _ = create_mesh_host(Identity.generate())
+ server.set_stream_handler(ECHO, echo)
+ async with server.run(listen_addrs=server_listen), client.run(listen_addrs=[]):
+ await client.connect(info_from_p2p_addr(multiaddr.Multiaddr(f"{server.get_addrs()[0]}")))
+ for i in range(300):
+ # Alternate the two ways a caller lets go of a stream.
+ with trio.fail_after(5):
+ stream = await open_stream(client, server.get_id(), ECHO, 5)
+ try:
+ await stream.write(b"ping")
+ assert await stream.read(4) == b"ping"
+ finally:
+ await (stream.close() if i % 2 else stream.reset())
+
+ trio.run(main)
+
+
+def test_open_stream_ends_at_its_timeout():
+ class Host:
+ async def new_stream(self, peer_id, protocols):
+ await trio.sleep_forever()
+
+ async def main():
+ started = trio.current_time()
+ with pytest.raises(ConnectionError, match="within 10s"):
+ await open_stream(Host(), ID.from_base58(ROUTER), ECHO, 10)
+ assert trio.current_time() - started == pytest.approx(10)
+
+ trio.run(main, clock=trio.testing.MockClock(autojump_threshold=0))
+
+
+def test_the_stream_slot_workaround_retires_with_libp2p_0_8():
+ """When the pin moves past 0.7, delete the Yamux.open_stream patch and
+ this flag in host.py, and the backlog test above."""
+ import importlib.metadata
+
+ major, minor = (int(p) for p in importlib.metadata.version("libp2p").split(".")[:2])
+ assert LIBP2P_LEAKS_STREAM_SLOTS == ((major, minor) < (0, 8))