Skip to content

Agent Mesh

sdk/python: refresh the credential from one thread at a time

Antonio Ojead68f2e35a69a
diff --git a/sdk/python/src/agent_mesh/mesh.py b/sdk/python/src/agent_mesh/mesh.py
index a810e326..2f29495c 100644
--- a/sdk/python/src/agent_mesh/mesh.py
+++ b/sdk/python/src/agent_mesh/mesh.py
@@ -57,6 +57,10 @@ class AgentMesh:
self.control_plane = control_plane
self._credential = credential
self._state_dir = state_dir
+ # refresh() and sync_control_plane() run in worker threads of the
+ # session's loops; the control plane redeems only the last biscuit it
+ # issued, and both write the same state files.
+ self._lock = threading.RLock()
@property
def peer_id(self) -> str:
@@ -173,29 +177,31 @@ class AgentMesh:
"""Trades the current biscuit for a fresh one and persists it. The control
plane redeems only the last biscuit it issued, so a lost refresh result
means re-enrolling; persisting before returning keeps that rare."""
- result = self.control_plane.refresh(self.identity, self._credential.biscuit)
- control_plane_keys = self._credential.control_plane_keys
- try:
- control_plane_keys = self.control_plane.keys(control_plane_keys)
- except Exception: # noqa: BLE001 - a failed /keys sync must not cost the new biscuit
- pass
- self._credential = replace(
- self._credential,
- biscuit=result.biscuit,
- expiration=result.expiration,
- control_plane_keys=control_plane_keys,
- issued_under_keys=list(control_plane_keys),
- )
- self.save()
- return self._credential
+ with self._lock:
+ result = self.control_plane.refresh(self.identity, self._credential.biscuit)
+ control_plane_keys = self._credential.control_plane_keys
+ try:
+ control_plane_keys = self.control_plane.keys(control_plane_keys)
+ except Exception: # noqa: BLE001 - a failed /keys sync must not cost the new biscuit
+ pass
+ self._credential = replace(
+ self._credential,
+ biscuit=result.biscuit,
+ expiration=result.expiration,
+ control_plane_keys=control_plane_keys,
+ issued_under_keys=list(control_plane_keys),
+ )
+ self.save()
+ return self._credential
def add_trusted_key(self, key: bytes) -> bool:
"""Adopts a signing key announced by a KEY_ROTATION event, so peers whose
credentials the new key signs verify before the next pull confirms it."""
- if any(bytes(k) == bytes(key) for k in self._credential.control_plane_keys):
- return False
- self._credential = replace(self._credential, control_plane_keys=[*self._credential.control_plane_keys, bytes(key)])
- return True
+ with self._lock:
+ if any(bytes(k) == bytes(key) for k in self._credential.control_plane_keys):
+ return False
+ self._credential = replace(self._credential, control_plane_keys=[*self._credential.control_plane_keys, bytes(key)])
+ return True
def sync_control_plane(self) -> ControlPlaneSync:
"""The member's pull from the control plane, as sam-node's SyncControlPlane:
@@ -205,6 +211,10 @@ class AgentMesh:
addresses and the ban set. Each part is attempted even when another
fails; the errors are reported together. Gossip events only bring this
forward; they are never the only way state arrives."""
+ with self._lock:
+ return self._sync_control_plane()
+
+ def _sync_control_plane(self) -> ControlPlaneSync:
errors: list[str] = []
keys_changed = False
refreshed = False
@@ -255,9 +265,10 @@ class AgentMesh:
"""Writes identity and credential to the state directory, if one is configured."""
if self._state_dir is None:
return
- self._state_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
- _write_atomic(self._state_dir / _IDENTITY_FILE, self.identity.to_libp2p_private_key())
- _write_atomic(self._state_dir / _CREDENTIAL_FILE, self._credential.to_json().encode())
+ with self._lock:
+ self._state_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
+ _write_atomic(self._state_dir / _IDENTITY_FILE, self.identity.to_libp2p_private_key())
+ _write_atomic(self._state_dir / _CREDENTIAL_FILE, self._credential.to_json().encode())
def _load_identity(state: Optional[Path]) -> Optional[Identity]:
diff --git a/sdk/python/tests/test_mesh.py b/sdk/python/tests/test_mesh.py
index 0af392d1..a00a69c1 100644
--- a/sdk/python/tests/test_mesh.py
+++ b/sdk/python/tests/test_mesh.py
@@ -12,8 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import base64
import json
import stat
+import threading
import time
import urllib.parse
from datetime import datetime, timezone
@@ -42,11 +44,22 @@ CP_KEY = Identity.generate()
class FakeControlPlane:
- """Approves everything and hands out numbered biscuits."""
+ """Approves everything and hands out numbered biscuits. With strict_refresh
+ it redeems only the last biscuit it issued, as the real one does, and
+ refresh_delay is how long a /refresh takes."""
- def __init__(self, keys_ok=True):
+ def __init__(self, keys_ok=True, strict_refresh=False, refresh_delay=0.0):
self.issued = 0
self.keys_ok = keys_ok
+ self.strict_refresh = strict_refresh
+ self.refresh_delay = refresh_delay
+ self.last_biscuit = b""
+ self._lock = threading.Lock()
+
+ def _issue(self, biscuit):
+ self.issued += 1
+ self.last_biscuit = biscuit
+ return biscuit
def _signed_keys(self):
ts = int(time.time() * 1000)
@@ -58,25 +71,28 @@ class FakeControlPlane:
if (method, path) == ("POST", "/register"):
# The biscuit names the JWT that was presented, so a test can see which.
jwt = pb.EnrollRequest.FromString(body).jwt
- self.issued += 1
return 200, pb.EnrollResponse(
- biscuit_token=f"biscuit-for-{jwt}".encode(),
+ biscuit_token=self._issue(f"biscuit-for-{jwt}".encode()),
control_plane_public_key=CP_KEY.public_key_raw,
router_addresses=["/dns4/router.example/tcp/4001/p2p/12D3KooWP8iKhDf3iCMo2H3butNVfdTUtYwYWYQ75jTGnynXPFMp"],
expire_time=_ts_s(int(time.time()) + 3600),
).SerializeToString()
if (method, path) == ("POST", "/enroll"):
- self.issued += 1
return 200, pb.BootstrapEnrollResponse(
status=pb.ENROLLMENT_STATUS_APPROVED,
- biscuit_token=f"biscuit-{self.issued}".encode(),
+ biscuit_token=self._issue(f"biscuit-{self.issued + 1}".encode()),
control_plane_public_key=CP_KEY.public_key_raw,
router_addresses=["/dns4/router.example/tcp/4001/p2p/12D3KooWP8iKhDf3iCMo2H3butNVfdTUtYwYWYQ75jTGnynXPFMp"],
expire_time=_ts_s(int(time.time()) + 3600),
).SerializeToString()
if (method, path) == ("POST", "/refresh"):
- self.issued += 1
- return 200, pb.TokenRefreshResponse(biscuit_token=f"biscuit-{self.issued}".encode(), expire_time=_ts_s(int(time.time()) + 7200)).SerializeToString()
+ time.sleep(self.refresh_delay)
+ with self._lock:
+ presented = base64.b64decode(headers["Authorization"].removeprefix("Bearer "))
+ if self.strict_refresh and presented != self.last_biscuit:
+ return 200, pb.TokenRefreshResponse(error_message="biscuit already redeemed").SerializeToString()
+ biscuit = self._issue(f"biscuit-{self.issued + 1}".encode())
+ return 200, pb.TokenRefreshResponse(biscuit_token=biscuit, expire_time=_ts_s(int(time.time()) + 7200)).SerializeToString()
if (method, path) == ("GET", "/keys"):
return (200, self._signed_keys().SerializeToString()) if self.keys_ok else (500, b"boom")
return 404, f"no route for {method} {path}".encode()
@@ -153,6 +169,34 @@ def test_enroll_without_state_dir_keeps_enrollment_key_when_keys_fails():
assert mesh.credential.biscuit == b"biscuit-2"
+def test_concurrent_refreshes_run_one_after_the_other(tmp_path):
+ """A session refreshes the credential from one worker thread and pulls
+ from the control plane, which may refresh too, from another. The control
+ plane redeems only the last biscuit it issued and both write the same
+ state files, so two refreshes at once leave the loser with a spent
+ biscuit, or renaming a temp file the winner already renamed."""
+ cp = FakeControlPlane(strict_refresh=True, refresh_delay=0.05)
+ state = tmp_path / "state"
+ mesh = AgentMesh.enroll("http://127.0.0.1:1", bootstrap_token="sbt_secret", state_dir=state, transport=cp.transport)
+ errors = []
+
+ def refresh():
+ try:
+ mesh.refresh()
+ except Exception as err: # noqa: BLE001 - collected for the assertion
+ errors.append(err)
+
+ threads = [threading.Thread(target=refresh) for _ in range(8)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ assert errors == []
+ assert mesh.credential.biscuit == b"biscuit-9"
+ assert json.loads((state / "credential.json").read_text())["biscuit"] == base64.b64encode(b"biscuit-9").decode()
+ assert AgentMesh.load(state, transport=cp.transport).credential.biscuit == b"biscuit-9"
+
+
def test_enroll_refuses_ambiguous_credentials():
cp = FakeControlPlane()
with pytest.raises(ValueError, match="exactly one of"):