Skip to content

Agent Mesh

sdk/python: admit every router, and reserve again when the connection goes

Antonio Ojeaddd83c449c4b
diff --git a/sdk/python/src/agent_mesh/session.py b/sdk/python/src/agent_mesh/session.py
index 640a7e86..ab983bc6 100644
--- a/sdk/python/src/agent_mesh/session.py
+++ b/sdk/python/src/agent_mesh/session.py
@@ -70,6 +70,11 @@ MIN_REFRESH_DELAY = 2.0
# go-libp2p's relay grants a reservation for an hour and drops it on expiry;
# its own clients renew two minutes before that.
RESERVATION_RENEW_LEAD = 2 * 60.0
+# The relay also drops the reservation when the connection goes, with a
+# router restart or a trimmed connection; this is how long the member is
+# unreachable through that router at most before it notices and reserves
+# again.
+RESERVATION_CHECK_INTERVAL = 30.0
# sam-node's --control-plane-sync-interval default.
DEFAULT_POLICY_SYNC = 15 * 60.0
# sam-node's --control-plane-sync-interval default, and its 2s first pull.
@@ -157,6 +162,7 @@ class MeshSession:
control_plane_sync_jitter: float = DEFAULT_CONTROL_PLANE_SYNC_JITTER
reservation_lead: float = RESERVATION_RENEW_LEAD
reservation_retry: float = DEFAULT_REFRESH_RETRY
+ reservation_check_interval: float = RESERVATION_CHECK_INTERVAL
_nursery: Optional[trio.Nursery] = field(default=None, repr=False)
_policy_rules: Optional[list[str]] = field(default=None, repr=False)
_sync_lock: trio.Lock = field(default_factory=trio.Lock, repr=False)
@@ -373,20 +379,24 @@ class MeshSession:
async def _reservation_loop(self) -> None:
"""Renews the relay reservation on each admitted router before the
- relay lets it expire. A member whose reservation lapsed still
- advertises the relayed address, and every dial to it fails with
- NO_RESERVATION. A router that dropped the connection forgets the
- admission with it, so that one is dialed and authenticated again
- before the reservation is asked for."""
+ relay lets it expire, and again as soon as the connection to that
+ router is found gone: the relay drops the reservation with the
+ connection, and a member that kept advertising the relayed address
+ would have every dial to it fail with NO_RESERVATION. A router that
+ dropped the connection forgets the admission with it, so that one is
+ dialed and authenticated again before the reservation is asked for."""
while True:
reserved = [r for r in self.routers if r.reservation is not None]
if not reserved:
return
due = min(r.reservation.expire for r in reserved) - self.reservation_lead - time.time() # type: ignore[union-attr]
- await trio.sleep(max(MIN_REFRESH_DELAY, due))
+ await trio.sleep(min(max(MIN_REFRESH_DELAY, due), self.reservation_check_interval))
failed = False
for i, r in enumerate(self.routers):
- if r.reservation is None or r.reservation.expire - time.time() > self.reservation_lead + MIN_REFRESH_DELAY:
+ if r.reservation is None:
+ continue
+ expiring = r.reservation.expire - time.time() <= self.reservation_lead + MIN_REFRESH_DELAY
+ if not expiring and ID.from_base58(r.peer_id) in self.host.get_connected_peers():
continue
try:
self.routers[i] = await _reserve_again(self.host, self.mesh, r)
@@ -510,6 +520,7 @@ async def join_mesh(
refresh_lead: float = DEFAULT_REFRESH_LEAD,
refresh_retry: float = DEFAULT_REFRESH_RETRY,
reservation_lead: float = RESERVATION_RENEW_LEAD,
+ reservation_check_interval: float = RESERVATION_CHECK_INTERVAL,
policy_sync_interval: float = DEFAULT_POLICY_SYNC,
control_plane_sync_interval: float = DEFAULT_CONTROL_PLANE_SYNC,
control_plane_sync_jitter: float = DEFAULT_CONTROL_PLANE_SYNC_JITTER,
@@ -555,6 +566,7 @@ async def join_mesh(
control_plane_sync_jitter=control_plane_sync_jitter,
reservation_lead=reservation_lead,
reservation_retry=refresh_retry,
+ reservation_check_interval=reservation_check_interval,
_nursery=nursery,
)
nursery.start_soon(session._events_loop, pubsub) # noqa: SLF001
@@ -572,6 +584,11 @@ async def join_mesh(
async def _admit(host: IHost, mesh: "AgentMesh", router_addrs: list[multiaddr.Multiaddr], reserve: bool) -> list[AdmittedRouter]:
+ """Authenticates with every router the control plane named, as the JS SDK
+ and sam-node do, and reserves a relay slot on the first that admits us.
+ A peer reserves on the first router of its own list, so a caller that
+ only knew one router could not reach a peer whose list started
+ elsewhere; `connect` tries the relayed path through each of these."""
admitted: list[AdmittedRouter] = []
failures: list[str] = []
for addr in router_addrs:
@@ -579,10 +596,8 @@ async def _admit(host: IHost, mesh: "AgentMesh", router_addrs: list[multiaddr.Mu
info = await peer_info(addr)
await dial(host, info)
credential = await _authenticate_router(host, mesh, info.peer_id)
- reservation = await reserve_relay(host, info.peer_id) if reserve else None
+ reservation = await reserve_relay(host, info.peer_id) if reserve and not any(r.reservation for r in admitted) else None
admitted.append(AdmittedRouter(peer_id=str(info.peer_id), addr=addr, credential=credential, reservation=reservation))
- if reserve:
- break
except Exception as err: # noqa: BLE001 - every router is tried, the summary names each failure
failures.append(f"{addr}: {err}")
if not admitted:
diff --git a/sdk/python/tests/test_session.py b/sdk/python/tests/test_session.py
index ef2184af..15e7cc00 100644
--- a/sdk/python/tests/test_session.py
+++ b/sdk/python/tests/test_session.py
@@ -235,6 +235,58 @@ def test_join_fails_when_the_relay_refuses_the_reservation():
trio.run(with_timeout, 30, main)
+def test_join_admits_every_router_and_reserves_on_the_first():
+ """A peer reserves its relay slot on the first router of its list; a
+ caller reaches it only through that router. So the member authenticates
+ with every router the control plane named and connect() can try each,
+ while one reservation is enough to be reachable."""
+
+ async def main():
+ async with trio.open_nursery() as nursery:
+ events = []
+ router_a, addr_a = await start_router(nursery, events=events)
+ router_b, addr_b = await start_router(nursery, events=events)
+ mesh = AgentMesh.enroll("http://127.0.0.1:1", bootstrap_token="sbt", transport=fake_control_plane([addr_a, addr_b]))
+ async with mesh.join(refresh_lead=0) as session:
+ assert [r.peer_id for r in session.routers] == [str(router_a.get_id()), str(router_b.get_id())]
+ assert [r.reservation is not None for r in session.routers] == [True, False]
+ assert session.relay_addresses == [f"{addr_a}/p2p-circuit/p2p/{mesh.peer_id}"]
+ assert events.count(("auth", mesh.peer_id)) == 2
+ assert events.count(("reserve", mesh.peer_id)) == 1
+ nursery.cancel_scope.cancel()
+
+ trio.run(with_timeout, 30, main)
+
+
+def test_a_dropped_router_connection_is_reserved_again_before_the_ttl():
+ """A router restart takes the reservation with the connection. The member
+ notices within the check interval and reserves again, authenticated
+ anew, long before the TTL would have had it renew."""
+
+ async def wait_for(predicate):
+ while not predicate():
+ await trio.sleep(0.1)
+
+ async def main():
+ async with trio.open_nursery() as nursery:
+ events = []
+ router, addr = await start_router(nursery, ttl=3600, events=events)
+ mesh = AgentMesh.enroll("http://127.0.0.1:1", bootstrap_token="sbt", transport=fake_control_plane([addr]))
+ async with mesh.join(refresh_lead=0, reservation_check_interval=0.5) as session:
+ member = session.host.get_id()
+ assert events == [("auth", mesh.peer_id), ("reserve", mesh.peer_id)]
+
+ await router.disconnect(member)
+ await wait_for(lambda: member not in router.get_connected_peers())
+ await wait_for(lambda: events.count(("reserve", mesh.peer_id)) >= 2)
+ assert events[2:] == [("auth", mesh.peer_id), ("reserve", mesh.peer_id)]
+ assert member in router.get_connected_peers()
+ assert session.relay_addresses == [f"{addr}/p2p-circuit/p2p/{mesh.peer_id}"]
+ nursery.cancel_scope.cancel()
+
+ trio.run(with_timeout, 60, main)
+
+
def test_join_renews_the_reservation_and_reauthenticates_after_a_disconnect():
"""go-libp2p's relay drops a reservation when its TTL passes and grants one
only to a peer authenticated on the current connection. The member renews