Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
sdk/python: return the yamux slot on a cancelled open, parse pre-release versions
Antonio Ojeaacb1e006149a
diff --git a/sdk/python/src/agent_mesh/host.py b/sdk/python/src/agent_mesh/host.pyindex 54d5e721..f23f072b 100644--- a/sdk/python/src/agent_mesh/host.py+++ b/sdk/python/src/agent_mesh/host.py@@ -20,6 +20,8 @@ from __future__ import annotationsimport importlib.metadataimport os+import re+import structfrom datetime import datetime, timedelta, timezonefrom typing import Sequence@@ -35,6 +37,8 @@ from libp2p.peer.id import IDfrom libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addrfrom libp2p.security.tls.transport import PROTOCOL_ID as TLS_PROTOCOL_IDfrom libp2p.security.tls.transport import IdentityConfig, TLSTransport+from libp2p.stream_muxer.exceptions import MuxedStreamError+from libp2p.stream_muxer.yamux.yamux import FLAG_SYN, TYPE_WINDOW_UPDATE, YAMUX_HEADER_FORMATfrom libp2p.stream_muxer.yamux.yamux import PROTOCOL_ID as YAMUX_PROTOCOL_IDfrom libp2p.stream_muxer.yamux.yamux import Yamux, YamuxStreamfrom multiaddr.resolvers import DNSResolver@@ -43,8 +47,10 @@ from .identity import Identitydef _libp2p_version() -> tuple[int, int]:- major, minor = importlib.metadata.version("libp2p").split(".")[:2]- return int(major), int(minor)+ # PEP 440 lets a pre-release follow the minor directly (0.8a1), so the+ # string is not split on dots.+ m = re.match(r"(\d+)\.(\d+)", importlib.metadata.version("libp2p"))+ return (int(m.group(1)), int(m.group(2))) if m else (0, 0)# py-libp2p 0.7 takes one of the 256 slots of a connection's@@ -52,47 +58,69 @@ def _libp2p_version() -> tuple[int, int]:# when sending the SYN fails, so after 256 streams on one connection every# open_stream blocks forever, silently. main releases the slot when the local# side closes the stream (libp2p/py-libp2p#1426); until that is released,-# Yamux.open_stream is replaced here to do the same. A subclass would not do:+# Yamux.open_stream is replaced here with the same steps plus that release,+# and the slot is returned on any failure, a trio.Cancelled after the acquire+# included, which 0.7 (except Exception) keeps. A subclass would not do:# MuxerMultistream.new_conn constructs Yamux by name, whatever muxer_opt says.-# Delete this block and its test with the libp2p>=0.8 bump.+# Delete this block and its tests with the libp2p>=0.8 bump.LIBP2P_LEAKS_STREAM_SLOTS = _libp2p_version() < (0, 8)-def _release_slot_on_close(open_stream): # type: ignore[no-untyped-def]- async def open_stream_releasing_slot(self: Yamux) -> YamuxStream:- stream = await open_stream(self)- release = self.stream_backlog_semaphore.release- close, reset = stream.close, stream.reset- released = False+async def _open_stream_returning_slot(self: Yamux) -> YamuxStream:+ await self.stream_backlog_semaphore.acquire()+ released = False- def release_once() -> None:- nonlocal released- if not released:- released = True- release()+ def release_once() -> None:+ nonlocal released+ if not released:+ released = True+ self.stream_backlog_semaphore.release()- async def close_and_release() -> None:- try:- await close()- finally:- release_once()-- async def reset_and_release() -> None:- try:- await reset()- finally:- release_once()-- stream.close = close_and_release # type: ignore[method-assign]- stream.reset = reset_and_release # type: ignore[method-assign]- return stream-- open_stream_releasing_slot.releases_slot = True # type: ignore[attr-defined]- return open_stream_releasing_slot---if LIBP2P_LEAKS_STREAM_SLOTS and not getattr(Yamux.open_stream, "releases_slot", False):- Yamux.open_stream = _release_slot_on_close(Yamux.open_stream) # type: ignore[method-assign]+ stream_id: int | None = None+ try:+ async with self.streams_lock:+ if self.event_shutting_down.is_set():+ raise MuxedStreamError("Connection is shutting down")+ stream_id = self.next_stream_id+ self.next_stream_id += 2+ stream = YamuxStream(stream_id, self, True)+ self.streams[stream_id] = stream+ self.stream_buffers[stream_id] = bytearray()+ self.stream_events[stream_id] = trio.Event()+ await self._write_frame(struct.pack(YAMUX_HEADER_FORMAT, 0, TYPE_WINDOW_UPDATE, FLAG_SYN, stream_id, 0))+ except BaseException as err:+ release_once()+ if stream_id is not None:+ with trio.CancelScope(shield=True):+ async with self.streams_lock:+ self.streams.pop(stream_id, None)+ self.stream_buffers.pop(stream_id, None)+ self.stream_events.pop(stream_id, None)+ if isinstance(err, Exception) and not isinstance(err, MuxedStreamError):+ raise MuxedStreamError(f"Failed to send SYN: {err}") from err+ raise++ close, reset = stream.close, stream.reset++ async def close_and_release() -> None:+ try:+ await close()+ finally:+ release_once()++ async def reset_and_release() -> None:+ try:+ await reset()+ finally:+ release_once()++ stream.close = close_and_release # type: ignore[method-assign]+ stream.reset = reset_and_release # type: ignore[method-assign]+ return stream+++if LIBP2P_LEAKS_STREAM_SLOTS:+ Yamux.open_stream = _open_stream_returning_slot # type: ignore[method-assign]def _certificate_template() -> x509.CertificateBuilder:diff --git a/sdk/python/tests/test_host.py b/sdk/python/tests/test_host.pyindex ac55f66c..13725a99 100644--- a/sdk/python/tests/test_host.py+++ b/sdk/python/tests/test_host.py@@ -18,6 +18,8 @@ address through its TXT records and keep the transports this host hasstream must do: come back after the 256th one on a connection (py-libp2p0.7 leaks its yamux backlog slots), or fail at a deadline."""+import struct+import multiaddrimport pytestimport trio@@ -25,6 +27,7 @@ import trio.testingfrom libp2p.custom_types import TProtocolfrom libp2p.peer.id import IDfrom libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addr+from libp2p.stream_muxer.yamux.yamux import FLAG_SYN, YAMUX_HEADER_FORMATfrom multiaddr.resolvers import DNSResolverfrom agent_mesh.host import LIBP2P_LEAKS_STREAM_SLOTS, create_mesh_host, dial_addrs, open_stream, peer_info@@ -154,6 +157,41 @@ def test_a_connection_outlives_the_yamux_stream_backlog():trio.run(main)+def test_a_stream_open_cut_by_its_deadline_returns_the_slot():+ """py-libp2p 0.7 releases the backlog slot only for an Exception; a+ trio.Cancelled from the caller's deadline, raised while the SYN waits on+ a stalled connection, kept it. Every such timeout was one slot fewer."""++ async def main():+ server, server_listen = create_mesh_host(Identity.generate(), ["/ip4/127.0.0.1/tcp/0"])+ client, _ = create_mesh_host(Identity.generate())+ server.set_stream_handler(ECHO, lambda stream: stream.close())+ async with server.run(listen_addrs=server_listen), client.run(listen_addrs=[]):+ await client.connect(info_from_p2p_addr(multiaddr.Multiaddr(f"{server.get_addrs()[0]}")))+ mux = client.get_network().connections[server.get_id()][0].muxed_conn+ slots = mux.stream_backlog_semaphore.value+ write_frame = mux._write_frame++ async def syn_stalls(header):+ # Only the SYN of a new stream; the muxer's pings and window updates pass.+ if struct.unpack(YAMUX_HEADER_FORMAT, header[:12])[2] & FLAG_SYN:+ await trio.sleep_forever()+ await write_frame(header)++ mux._write_frame = syn_stalls+ for _ in range(3):+ with pytest.raises(ConnectionError, match="within 0.2s"):+ await open_stream(client, server.get_id(), ECHO, 0.2)+ mux._write_frame = write_frame+ assert mux.stream_backlog_semaphore.value == slots+ with trio.fail_after(5):+ stream = await open_stream(client, server.get_id(), ECHO, 5)+ await stream.close()+ assert mux.stream_backlog_semaphore.value == slots++ trio.run(main)++def test_open_stream_ends_at_its_timeout():class Host:async def new_stream(self, peer_id, protocols):