Skip to content

Agent Mesh

feat(agentmesh-cop): watch the mesh as a member through the Python SDK

Tomas Tormo92f4d96d876f
diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml
index e9993ddf..05659f01 100644
--- a/.github/workflows/sdk.yml
+++ b/.github/workflows/sdk.yml
@@ -24,6 +24,7 @@ on:
branches: [main]
paths:
- 'sdk/**'
+ - 'development/tools/agentmesh-cop/**'
- 'api/agentmesh.proto'
- 'hack/gen-sdk-proto.sh'
- 'hack/gen-sdk-docs/**'
@@ -36,6 +37,7 @@ on:
branches: [main]
paths:
- 'sdk/**'
+ - 'development/tools/agentmesh-cop/**'
- 'api/agentmesh.proto'
- 'hack/gen-sdk-proto.sh'
- 'hack/gen-sdk-docs/**'
diff --git a/Makefile b/Makefile
index e3ea9e65..c1b0c5ec 100644
--- a/Makefile
+++ b/Makefile
@@ -166,7 +166,8 @@ e2e-test: build docker-build
# tests and leave it installed, which is what TestNativeSDKs needs to run
# both against a real control plane instead of skipping. sdk-js also
# compiles the example programs the docs embed; TestNativeSDKExamples runs
-# them.
+# them. sdk-python also runs agentmesh-cop's tests, since the cop runs on
+# the Python SDK.
.PHONY: sdk-js sdk-python sdk-proto sdk-docs sdk-test
sdk-js:
cd sdk/js && npm ci --no-fund --no-audit && npm test && npm run build && npm run examples
@@ -175,6 +176,7 @@ sdk-python:
python3 -m venv sdk/python/.venv
./sdk/python/.venv/bin/pip install -q -e './sdk/python[test,examples]'
./sdk/python/.venv/bin/python -m pytest -q sdk/python/tests
+ ./sdk/python/.venv/bin/python -m pytest -q development/tools/agentmesh-cop
sdk-proto:
./hack/gen-sdk-proto.sh
diff --git a/development/tools/agentmesh-cop/agentmesh_cop.py b/development/tools/agentmesh-cop/agentmesh_cop.py
new file mode 100644
index 00000000..1af3ecff
--- /dev/null
+++ b/development/tools/agentmesh-cop/agentmesh_cop.py
@@ -0,0 +1,403 @@
+"""agentmesh-cop: a mesh member that watches the mesh and alerts on trouble.
+
+It joins with the native Python SDK and, every poll, pulls the control plane
+(signing keys, routers, bans), checks which listed routers admit it, and looks
+the services up in the DHT. It holds a session with every router that admits
+it and reserves no relay slot, since nothing needs to reach it.
+
+Env: AGENTMESH_CONTROL_PLANE_URL; AGENTMESH_BOOTSTRAP_TOKEN_PATH or
+AGENTMESH_JWT_PATH (first run only, see the SDK); AGENTMESH_STATE_DIR;
+AGENTMESH_INSECURE_CONTROL_PLANE; POLL_INTERVAL, MISS_THRESHOLD; and the
+channel settings in channels.py.
+"""
+
+import json
+import logging
+import os
+import re
+import sys
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Dict, List, Optional, Tuple
+
+import trio
+from agent_mesh.session import canonical_peer_ids
+
+from channels import Channel, build_channels
+
+logger = logging.getLogger("agentmesh-cop")
+
+
+@dataclass
+class AgentMeshCopConfig:
+ poll_interval: float
+ miss_threshold: int
+
+
+def load_config(env) -> AgentMeshCopConfig:
+ return AgentMeshCopConfig(
+ poll_interval=float(env.get("POLL_INTERVAL", "30")),
+ miss_threshold=int(env.get("MISS_THRESHOLD", "3")),
+ )
+
+
+@dataclass
+class Alert:
+ severity: str
+ title: str
+ description: str
+ peer_id: str = ""
+ service: str = ""
+
+
+SEVERITY_EMOJI = {"critical": "🚨", "warning": "⚠️", "info": "ℹ️"}
+
+CONTROL_CHARS_RE = re.compile(r"[\x00-\x1f\x7f]+")
+
+
+def sanitize_alert_text(text: str, limit: int = 200) -> str:
+ """Strips control chars (incl. newlines) so attacker-controlled strings can't forge alert lines."""
+ cleaned = CONTROL_CHARS_RE.sub(" ", text).strip()
+ if len(cleaned) > limit:
+ cleaned = cleaned[:limit].rstrip() + "..."
+ return cleaned
+
+
+def format_alert(alert: Alert, cop_peer_id: str, timestamp: str) -> str:
+ title = sanitize_alert_text(alert.title)
+ lines = [f"{SEVERITY_EMOJI[alert.severity]} [{alert.severity.upper()}] {title}"]
+ if alert.description:
+ lines.append(sanitize_alert_text(alert.description))
+ if alert.peer_id:
+ lines.append(f"peer: {sanitize_alert_text(alert.peer_id)}")
+ if alert.service:
+ lines.append(f"service: {sanitize_alert_text(alert.service)}")
+ lines.append(f"reported by {cop_peer_id} at {timestamp}")
+ return "\n".join(lines)
+
+
+def delivery_error(error: Exception) -> str:
+ # Channel URLs carry secrets (the Telegram bot token, the Slack webhook) and httpx errors quote the URL.
+ response = getattr(error, "response", None)
+ return f"HTTP {response.status_code}" if response is not None else type(error).__name__
+
+
+async def deliver(channels: List[Channel], message: str) -> None:
+ async def deliver_to_channel(channel: Channel) -> None:
+ for attempt in range(3):
+ try:
+ await channel.send(message)
+ return
+ except Exception as error:
+ if attempt == 2:
+ print(f"[-] {channel.name} delivery failed after 3 attempts: {delivery_error(error)}", flush=True)
+ else:
+ await trio.sleep(2**attempt)
+
+ # Concurrent so one slow or failing channel's retries don't delay the others.
+ async with trio.open_nursery() as nursery:
+ for channel in channels:
+ nursery.start_soon(deliver_to_channel, channel)
+
+
+@dataclass
+class MeshSnapshot:
+ """One poll's view of the mesh. None means that part could not be read this poll."""
+
+ sync_errors: List[str]
+ listed_routers: Optional[set]
+ signing_keys: set
+ banned_peers: Optional[set]
+ held_routers: set
+ services: Optional[set]
+
+
+@dataclass
+class AgentMeshCopState:
+ control_plane_misses: int = 0
+ listed_routers: Optional[set] = None
+ signing_keys: Optional[set] = None
+ banned_peers: Optional[set] = None
+ unreachable_polls: dict = field(default_factory=dict)
+ unreachable_routers: set = field(default_factory=set)
+ isolated_polls: int = 0
+ isolation: Optional[str] = None # title of the isolation alert in force
+ baseline: set = field(default_factory=set)
+ miss_counts: dict = field(default_factory=dict)
+ first_snapshot: bool = True
+
+
+def diff_alerts(previous: Optional[set], current: set, appeared: Alert, disappeared: Alert,
+ describe=lambda item: item) -> List[Alert]:
+ """One alert per item added to or dropped from a set; the first sighting is the baseline."""
+ if previous is None:
+ return []
+ alerts = []
+ for item in sorted(current - previous):
+ alerts.append(Alert(appeared.severity, appeared.title, appeared.description, peer_id=describe(item)))
+ for item in sorted(previous - current):
+ alerts.append(Alert(disappeared.severity, disappeared.title, disappeared.description, peer_id=describe(item)))
+ return alerts
+
+
+def detect_control_plane(state: AgentMeshCopState, snapshot: MeshSnapshot, miss_threshold: int) -> List[Alert]:
+ alerts = []
+ if snapshot.sync_errors:
+ state.control_plane_misses += 1
+ if state.control_plane_misses == miss_threshold:
+ alerts.append(Alert("warning", "control plane sync failing",
+ f"{miss_threshold} consecutive pulls failed: {'; '.join(snapshot.sync_errors)}"))
+ else:
+ if state.control_plane_misses >= miss_threshold:
+ alerts.append(Alert("info", "control plane sync recovered",
+ f"pull succeeded after {state.control_plane_misses} failures"))
+ state.control_plane_misses = 0
+
+ alerts.extend(diff_alerts(state.signing_keys, snapshot.signing_keys,
+ Alert("critical", "new control-plane signing key", "a key the cop did not trust before"),
+ Alert("warning", "control-plane signing key retired", "a trusted key left /keys"),
+ describe=lambda key: f"key {key[:16]}..."))
+ state.signing_keys = snapshot.signing_keys
+
+ if snapshot.listed_routers is not None:
+ alerts.extend(diff_alerts(state.listed_routers, snapshot.listed_routers,
+ Alert("info", "router listed", "the control plane leased a new router"),
+ Alert("warning", "router delisted", "the router's lease expired or was dropped")))
+ state.listed_routers = snapshot.listed_routers
+
+ if snapshot.banned_peers is not None:
+ alerts.extend(diff_alerts(state.banned_peers, snapshot.banned_peers,
+ Alert("critical", "peer banned", "the control plane banned this peer"),
+ Alert("info", "peer unbanned", "the control plane lifted the ban")))
+ state.banned_peers = snapshot.banned_peers
+ return alerts
+
+
+ISOLATION_RECOVERY = {"cop isolated": "cop reconnected", "mesh unreachable": "mesh reachable again"}
+
+
+def detect_router_reachability(state: AgentMeshCopState, snapshot: MeshSnapshot, miss_threshold: int) -> List[Alert]:
+ """Routers the control plane lists against the routers that admit the cop: a listed
+ router that refuses a member refuses every new agent too."""
+ # Before the first router list any held router counts, so a cop cut off from the start still alerts.
+ held = snapshot.held_routers if state.listed_routers is None else snapshot.held_routers & state.listed_routers
+ listed = state.listed_routers or set()
+ alerts = []
+ if not held:
+ state.isolated_polls += 1
+ if state.isolated_polls >= miss_threshold:
+ if state.control_plane_misses >= miss_threshold:
+ # Nothing answers the cop at all: its own network is the likelier fault.
+ verdict = Alert("warning", "cop isolated",
+ "neither the control plane nor any listed router answers this member")
+ else:
+ verdict = Alert("critical", "mesh unreachable",
+ f"the control plane lists {len(listed)} router(s) and none admits a member")
+ if verdict.title != state.isolation:
+ alerts.append(verdict)
+ state.isolation = verdict.title
+ return alerts
+ if state.isolation is not None:
+ alerts.append(Alert("info", ISOLATION_RECOVERY[state.isolation],
+ f"{len(held)} router(s) admit a member"))
+ state.isolation = None
+ state.isolated_polls = 0
+
+ for router in sorted(listed - held):
+ polls = state.unreachable_polls.get(router, 0) + 1
+ state.unreachable_polls[router] = polls
+ if polls == miss_threshold:
+ state.unreachable_routers.add(router)
+ alerts.append(Alert("warning", "router unreachable",
+ f"listed by the control plane but refused or dropped a member for {polls} polls",
+ peer_id=router))
+ for router in sorted(state.unreachable_routers & held):
+ alerts.append(Alert("info", "router reachable again", "the router admits a member again", peer_id=router))
+ state.unreachable_routers &= listed - held
+ state.unreachable_polls = {router: polls for router, polls in state.unreachable_polls.items()
+ if router in listed - held}
+ return alerts
+
+
+def detect_churn(state: AgentMeshCopState, services_snapshot: Optional[set], miss_threshold: int) -> List[Alert]:
+ if services_snapshot is None or state.isolation is not None:
+ return []
+ if state.first_snapshot:
+ state.baseline = set(services_snapshot)
+ state.miss_counts = {}
+ state.first_snapshot = False
+ return []
+
+ alerts = []
+ for entry in sorted(services_snapshot - state.baseline):
+ service_type, service_name, peer_id = entry
+ alerts.append(Alert("info", "service appeared", f"{service_type}://{service_name} advertised on the mesh",
+ peer_id=peer_id, service=f"{service_type}://{service_name}"))
+ state.baseline.add(entry)
+ for entry in list(state.miss_counts):
+ if entry in services_snapshot:
+ del state.miss_counts[entry]
+ for entry in sorted(state.baseline - services_snapshot):
+ misses = state.miss_counts.get(entry, 0) + 1
+ if misses >= miss_threshold:
+ service_type, service_name, peer_id = entry
+ alerts.append(Alert("warning", "service disappeared",
+ f"{service_type}://{service_name} missing for {misses} consecutive polls",
+ peer_id=peer_id, service=f"{service_type}://{service_name}"))
+ state.baseline.discard(entry)
+ state.miss_counts.pop(entry, None)
+ else:
+ state.miss_counts[entry] = misses
+ return alerts
+
+
+def evaluate_cycle(state: AgentMeshCopState, snapshot: MeshSnapshot, config: AgentMeshCopConfig):
+ alerts = []
+ alerts.extend(detect_control_plane(state, snapshot, config.miss_threshold))
+ alerts.extend(detect_router_reachability(state, snapshot, config.miss_threshold))
+ alerts.extend(detect_churn(state, snapshot.services, config.miss_threshold))
+ return state, alerts
+
+
+SERVICE_TYPES = ["mcp", "inference", "a2a", "egress"]
+DISCOVERY_LIMIT = 200
+# ServiceType numbers in api/agentmesh.proto, as list_local_services renders them.
+SERVICE_TYPE_NAMES = {1: "mcp", 2: "inference", 3: "a2a", 4: "egress"}
+PROVIDER_QUERY_TIMEOUT = 10.0
+
+
+async def provider_services(session, provider) -> List[Tuple[str, str]]:
+ """The (type, name) pairs a provider registered, from its catalog's list_local_services."""
+ result = await session.call_tool(provider, "", "list_local_services", {})
+ if result.is_error or not result.text:
+ raise RuntimeError(f"list_local_services failed: {result.text}")
+ return [(SERVICE_TYPE_NAMES.get(service.get("type")), service.get("name", ""))
+ for service in json.loads(result.text[0])]
+
+
+async def fetch_services(session) -> set:
+ # The DHT names the providers of a type, not their services; each provider names its own.
+ providers: Dict[str, object] = {}
+ advertised_types: Dict[str, set] = {}
+ for service_type in SERVICE_TYPES:
+ for provider in await session.discover(service_type, limit=DISCOVERY_LIMIT):
+ providers[provider.peer_id] = provider
+ advertised_types.setdefault(provider.peer_id, set()).add(service_type)
+ snapshot = set()
+ for peer_id, provider in providers.items():
+ try:
+ with trio.fail_after(PROVIDER_QUERY_TIMEOUT):
+ services = await provider_services(session, provider)
+ except Exception as error:
+ # Left out of this poll: its services count as missed, so only a lasting silence alerts.
+ # Debug only, since a dead provider's DHT record keeps it here for hours.
+ logger.debug("provider %s did not list its services: %s", peer_id, error)
+ continue
+ # A node lists every service it registered, advertised in the DHT or not.
+ snapshot.update((service_type, name, peer_id) for service_type, name in services
+ if service_type in advertised_types[peer_id])
+ return snapshot
+
+
+async def take_snapshot(session) -> MeshSnapshot:
+ # The SDK keeps away from a failed router for minutes; the cop probes every listed router each poll.
+ session.shunned_routers.clear()
+ result = await session.sync()
+ # Raw wire strings; the session keys its routers on the canonical form, so compare on that.
+ listed_routers = (set(canonical_peer_ids([router.peer_id for router in result.routers]))
+ if result.routers is not None else None)
+ banned_peers = set(canonical_peer_ids(result.banned_peer_ids)) if result.banned_peer_ids is not None else None
+ # session.routers keeps a dead router until its redials run out; count only live connections.
+ connected = {str(peer) for peer in session.host.get_connected_peers()}
+ held_routers = {router.peer_id for router in session.routers if router.peer_id in connected}
+ services = None
+ if held_routers:
+ try:
+ services = await fetch_services(session)
+ except Exception as error:
+ print(f"[-] service discovery failed: {error}", flush=True)
+ return MeshSnapshot(
+ sync_errors=list(result.errors),
+ listed_routers=listed_routers,
+ signing_keys={bytes(key).hex() for key in session.mesh.credential.control_plane_keys},
+ banned_peers=banned_peers,
+ held_routers=held_routers,
+ services=services,
+ )
+
+
+async def alert_all(channels: List[Channel], alerts: List[Alert], cop_peer_id: str) -> None:
+ timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")
+ for alert in alerts:
+ await deliver(channels, format_alert(alert, cop_peer_id, timestamp))
+
+
+# More routers than any mesh lists: the session then holds every router that admits it,
+# and its own loop redials the ones that drop and retries the ones it shunned.
+HOLD_EVERY_ROUTER = 10_000
+
+
+async def watch(session, channels: List[Channel], config: AgentMeshCopConfig) -> None:
+ state = AgentMeshCopState()
+ while True:
+ try:
+ state, alerts = evaluate_cycle(state, await take_snapshot(session), config)
+ await alert_all(channels, alerts, session.peer_id)
+ except Exception as error:
+ print(f"[-] poll cycle failed: {error}", flush=True)
+ await trio.sleep(config.poll_interval)
+
+
+async def run_agentmesh_cop(enroll, config: AgentMeshCopConfig, channels: List[Channel]) -> None:
+ mesh = None
+ join_failed = False
+ while True:
+ try:
+ if mesh is None:
+ # Blocks on HTTP and, without auto-approve, on an operator, so it runs off the trio thread.
+ mesh = await trio.to_thread.run_sync(enroll)
+ # The cop drives every pull itself; key rotation events still bring one forward.
+ async with mesh.join(routers=HOLD_EVERY_ROUTER, reserve=False, control_plane_sync_interval=0) as session:
+ print(f"[+] on the mesh as {session.peer_id}, holding {len(session.routers)} router(s)", flush=True)
+ if join_failed:
+ await alert_all(channels, [Alert("info", "joined the mesh again", "")], session.peer_id)
+ join_failed = False
+ await watch(session, channels, config)
+ except Exception as error:
+ print(f"[-] cannot join the mesh: {error}", flush=True)
+ if not join_failed:
+ reporter = mesh.peer_id if mesh is not None else "a cop that is not enrolled yet"
+ await alert_all(channels, [Alert("critical", "cannot join the mesh", str(error))], reporter)
+ join_failed = True
+ await trio.sleep(config.poll_interval)
+
+
+def main() -> None:
+ from agent_mesh import AgentMesh
+
+ # Logs every refused dial, some with a full traceback; the cop reports reachability itself.
+ logging.getLogger("libp2p.transport").setLevel(logging.CRITICAL)
+ config = load_config(os.environ)
+ channels = build_channels(os.environ)
+ channel_names = ", ".join(channel.name for channel in channels)
+ print(f"[*] agentmesh-cop starting: poll={config.poll_interval}s miss_threshold={config.miss_threshold} "
+ f"channels=[{channel_names}]", flush=True)
+ control_plane_url = os.environ["AGENTMESH_CONTROL_PLANE_URL"]
+
+ def enroll():
+ return AgentMesh.enroll(
+ control_plane_url,
+ bootstrap_token_path=os.environ.get("AGENTMESH_BOOTSTRAP_TOKEN_PATH"),
+ jwt_path=os.environ.get("AGENTMESH_JWT_PATH"),
+ state_dir=os.environ.get("AGENTMESH_STATE_DIR", "~/.config/agentmesh/cop"),
+ allow_insecure=os.environ.get("AGENTMESH_INSECURE_CONTROL_PLANE") == "true",
+ )
+
+ trio.run(run_agentmesh_cop, enroll, config, channels)
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except KeyboardInterrupt:
+ sys.exit(0)
diff --git a/development/tools/agentmesh-cop/channels.py b/development/tools/agentmesh-cop/channels.py
new file mode 100644
index 00000000..78e576ff
--- /dev/null
+++ b/development/tools/agentmesh-cop/channels.py
@@ -0,0 +1,67 @@
+import abc
+from typing import List
+
+import httpx
+
+
+class Channel(abc.ABC):
+ """Delivery backend. Subclasses self-register; declare required_env and build_channels() picks them up."""
+
+ name = "channel"
+ required_env: tuple = ()
+ registry: list = []
+
+ def __init_subclass__(cls, **kwargs):
+ super().__init_subclass__(**kwargs)
+ Channel.registry.append(cls)
+
+ @classmethod
+ def from_env(cls, env):
+ # __init__ parameters must line up with required_env order.
+ return cls(*(env[var] for var in cls.required_env))
+
+ @abc.abstractmethod
+ async def send(self, message: str) -> None: ...
+
+
+class SlackChannel(Channel):
+ name = "slack"
+ required_env = ("SLACK_WEBHOOK_URL",)
+
+ def __init__(self, webhook_url: str):
+ self.webhook_url = webhook_url
+
+ async def send(self, message: str) -> None:
+ async with httpx.AsyncClient() as http_client:
+ response = await http_client.post(self.webhook_url, json={"text": message}, timeout=10.0)
+ response.raise_for_status()
+
+
+class TelegramChannel(Channel):
+ name = "telegram"
+ required_env = ("TELEGRAM_BOT_TOKEN", "TELEGRAM_CHAT_ID")
+
+ def __init__(self, bot_token: str, chat_id: str):
+ self.bot_token = bot_token
+ self.chat_id = chat_id
+
+ async def send(self, message: str) -> None:
+ url = f"https://api.telegram.org/bot{self.bot_token}/sendMessage"
+ async with httpx.AsyncClient() as http_client:
+ response = await http_client.post(url, json={"chat_id": self.chat_id, "text": message}, timeout=10.0)
+ response.raise_for_status()
+
+
+class StdoutChannel(Channel):
+ name = "stdout"
+
+ async def send(self, message: str) -> None:
+ print(f"[ALERT] {message}", flush=True)
+
+
+def build_channels(env) -> List[Channel]:
+ # Stdout is always on so the log shows every alert, delivered or not.
+ channels = [cls.from_env(env) for cls in Channel.registry
+ if cls.required_env and all(env.get(var) for var in cls.required_env)]
+ channels.append(StdoutChannel())
+ return channels
diff --git a/development/tools/agentmesh-cop/test_agentmesh_cop.py b/development/tools/agentmesh-cop/test_agentmesh_cop.py
new file mode 100644
index 00000000..63bf9867
--- /dev/null
+++ b/development/tools/agentmesh-cop/test_agentmesh_cop.py
@@ -0,0 +1,444 @@
+import json
+from dataclasses import dataclass, field
+from types import SimpleNamespace
+from typing import List, Optional
+
+import httpx
+import trio
+import trio.testing
+
+from agentmesh_cop import (
+ AgentMeshCopConfig,
+ AgentMeshCopState,
+ Alert,
+ MeshSnapshot,
+ deliver,
+ evaluate_cycle,
+ fetch_services,
+ format_alert,
+ load_config,
+ run_agentmesh_cop,
+ sanitize_alert_text,
+ take_snapshot,
+)
+from channels import (
+ Channel,
+ SlackChannel,
+ StdoutChannel,
+ TelegramChannel,
+ build_channels,
+)
+
+
+def test_load_config_defaults():
+ config = load_config({})
+ assert config.poll_interval == 30.0
+ assert config.miss_threshold == 3
+
+
+def test_load_config_overrides():
+ config = load_config({"POLL_INTERVAL": "5", "MISS_THRESHOLD": "2"})
+ assert config.poll_interval == 5.0
+ assert config.miss_threshold == 2
+
+
+def test_build_channels_stdout_fallback():
+ channels = build_channels({})
+ assert len(channels) == 1
+ assert isinstance(channels[0], StdoutChannel)
+
+
+def test_build_channels_slack_and_telegram_keep_stdout():
+ env = {
+ "SLACK_WEBHOOK_URL": "https://hooks.slack.example/x",
+ "TELEGRAM_BOT_TOKEN": "token",
+ "TELEGRAM_CHAT_ID": "42",
+ }
+ channels = build_channels(env)
+ assert {type(c) for c in channels} == {SlackChannel, TelegramChannel, StdoutChannel}
+
+
+def test_build_channels_telegram_requires_chat_id():
+ channels = build_channels({"TELEGRAM_BOT_TOKEN": "token"})
+ assert len(channels) == 1
+ assert isinstance(channels[0], StdoutChannel)
+
+
+def test_build_channels_picks_up_registered_subclass():
+ class EchoChannel(Channel):
+ name = "echo"
+ required_env = ("ECHO_TARGET",)
+
+ def __init__(self, target):
+ self.target = target
+
+ async def send(self, message):
+ pass
+
+ try:
+ channels = build_channels({"ECHO_TARGET": "x"})
+ assert [type(c) for c in channels] == [EchoChannel, StdoutChannel]
+ assert channels[0].target == "x"
+ finally:
+ Channel.registry.remove(EchoChannel)
+
+
+def test_format_alert():
+ alert = Alert("critical", "peer banned", "the control plane banned this peer", peer_id="12D3KooPeer")
+ message = format_alert(alert, "12D3KooCop", "2026-08-06T10:00:00+00:00")
+ assert message.startswith("🚨 [CRITICAL] peer banned")
+ assert "the control plane banned this peer" in message
+ assert "peer: 12D3KooPeer" in message
+ assert "reported by 12D3KooCop at 2026-08-06T10:00:00+00:00" in message
+
+
+class FlakyChannel(Channel):
+ name = "flaky"
+
+ def __init__(self, failures):
+ self.failures = failures
+ self.attempts = 0
+ self.delivered = []
+
+ async def send(self, message):
+ self.attempts += 1
+ if self.attempts <= self.failures:
+ raise RuntimeError("boom")
+ self.delivered.append(message)
+
+
+def test_deliver_retries_then_succeeds():
+ channel = FlakyChannel(failures=2)
+ trio.run(deliver, [channel], "hello")
+ assert channel.delivered == ["hello"]
+ assert channel.attempts == 3
+
+
+def test_deliver_failure_does_not_log_the_channel_secret(capsys):
+ class RejectingChannel(Channel):
+ name = "telegram"
+
+ async def send(self, message):
+ request = httpx.Request("POST", "https://api.telegram.org/botSECRET-TOKEN/sendMessage")
+ response = httpx.Response(401, request=request)
+ response.raise_for_status()
+
+ trio.run(deliver, [RejectingChannel()], "hello")
+ output = capsys.readouterr().out
+ assert "HTTP 401" in output
+ assert "SECRET-TOKEN" not in output
+
+
+def test_deliver_gives_up_without_raising():
+ channel = FlakyChannel(failures=5)
+ trio.run(deliver, [channel], "hello")
+ assert channel.delivered == []
+ assert channel.attempts == 3
+
+
+def test_sanitize_alert_text_strips_control_chars_and_collapses():
+ hostile = "svc\n🚨 [CRITICAL] fake alert\x07more"
+ sanitized = sanitize_alert_text(hostile)
+ assert "\n" not in sanitized
+ assert "\x07" not in sanitized
+ assert "svc" in sanitized and "fake alert" in sanitized
+
+
+def test_sanitize_alert_text_truncates_with_ellipsis():
+ truncated = sanitize_alert_text("a" * 500, limit=50)
+ assert len(truncated) == 53
+ assert truncated.endswith("...")
+
+
+def test_format_alert_sanitizes_hostile_service_name():
+ hostile_service = "svc\n🚨 [CRITICAL] fake peer banned"
+ alert = Alert("info", "service appeared", "desc", peer_id="peer-1", service=hostile_service)
+ message = format_alert(alert, "12D3KooCop", "2026-08-06T10:00:00+00:00")
+ lines = message.split("\n")
+ assert len(lines) == 5 # no forged extra line from the injected newline
+ assert lines[0] == "ℹ️ [INFO] service appeared"
+ assert "svc" in lines[3] and "CRITICAL" in lines[3]
+
+
+CONFIG = AgentMeshCopConfig(poll_interval=30.0, miss_threshold=3)
+ROUTERS = {"router-a", "router-b"}
+KEYS = {"aa" * 32}
+SERVICE_A = ("mcp", "github", "peer-1")
+SERVICE_B = ("inference", "laptop-llm", "peer-2")
+
+
+def snapshot(errors=(), listed=ROUTERS, keys=KEYS, banned=frozenset(), held=ROUTERS, services=frozenset()):
+ return MeshSnapshot(
+ sync_errors=list(errors),
+ listed_routers=None if listed is None else set(listed),
+ signing_keys=set(keys),
+ banned_peers=None if banned is None else set(banned),
+ held_routers=set(held),
+ services=None if services is None else set(services),
+ )
+
+
+def cycle(state, **kwargs):
+ return evaluate_cycle(state, snapshot(**kwargs), CONFIG)
+
+
+def titles(alerts):
+ return [(a.severity, a.title, a.peer_id) for a in alerts]
+
+
+def test_first_poll_is_a_baseline():
+ _, alerts = cycle(AgentMeshCopState(), services={SERVICE_A})
+ assert alerts == []
+
+
+def test_service_provider_appears_and_disappears_after_misses():
+ state, _ = cycle(AgentMeshCopState(), services={SERVICE_A})
+ state, alerts = cycle(state, services={SERVICE_A, SERVICE_B})
+ assert titles(alerts) == [("info", "service appeared", "peer-2")]
+ assert alerts[0].service == "inference://laptop-llm"
+ for _ in range(2):
+ state, alerts = cycle(state, services={SERVICE_B})
+ assert alerts == []
+ state, alerts = cycle(state, services={SERVICE_B})
+ assert titles(alerts) == [("warning", "service disappeared", "peer-1")]
+ assert alerts[0].service == "mcp://github"
+
+
+def test_signing_key_added_and_retired():
+ state, _ = cycle(AgentMeshCopState())
+ state, alerts = cycle(state, keys=KEYS | {"bb" * 32})
+ assert titles(alerts) == [("critical", "new control-plane signing key", "key bbbbbbbbbbbbbbbb...")]
+ state, alerts = cycle(state, keys={"bb" * 32})
+ assert titles(alerts) == [("warning", "control-plane signing key retired", "key aaaaaaaaaaaaaaaa...")]
+
+
+def test_router_listed_and_delisted():
+ state, _ = cycle(AgentMeshCopState())
+ state, alerts = cycle(state, listed={"router-a", "router-c"}, held={"router-a", "router-c"})
+ assert titles(alerts) == [("info", "router listed", "router-c"), ("warning", "router delisted", "router-b")]
+
+
+def test_failed_info_keeps_the_last_router_list():
+ state, _ = cycle(AgentMeshCopState())
+ state, alerts = cycle(state, listed=None, banned=None)
+ assert alerts == []
+ assert state.listed_routers == ROUTERS
+
+
+def test_peer_banned_and_unbanned():
+ state, _ = cycle(AgentMeshCopState())
+ state, alerts = cycle(state, banned={"peer-x"})
+ assert titles(alerts) == [("critical", "peer banned", "peer-x")]
+ state, alerts = cycle(state, banned=set())
+ assert titles(alerts) == [("info", "peer unbanned", "peer-x")]
+
+
+def test_control_plane_failing_alerts_once_at_threshold_then_recovers():
+ state, _ = cycle(AgentMeshCopState())
+ for _ in range(2):
+ state, alerts = cycle(state, errors=["info: timed out"], listed=None, banned=None)
+ assert alerts == []
+ state, alerts = cycle(state, errors=["info: timed out"], listed=None, banned=None)
+ assert titles(alerts) == [("warning", "control plane sync failing", "")]
+ assert "info: timed out" in alerts[0].description
+ state, alerts = cycle(state, errors=["info: timed out"], listed=None, banned=None)
+ assert alerts == []
+ state, alerts = cycle(state)
+ assert titles(alerts) == [("info", "control plane sync recovered", "")]
+
+
+def test_listed_router_that_refuses_a_member_alerts_after_misses_then_recovers():
+ state, _ = cycle(AgentMeshCopState())
+ for _ in range(2):
+ state, alerts = cycle(state, held={"router-a"})
+ assert alerts == []
+ state, alerts = cycle(state, held={"router-a"})
+ assert titles(alerts) == [("warning", "router unreachable", "router-b")]
+ state, alerts = cycle(state, held={"router-a"})
+ assert alerts == []
+ state, alerts = cycle(state)
+ assert titles(alerts) == [("info", "router reachable again", "router-b")]
+
+
+def test_no_router_admits_while_control_plane_answers_is_a_mesh_outage():
+ state, _ = cycle(AgentMeshCopState())
+ for _ in range(2):
+ state, alerts = cycle(state, held=set(), services=None)
+ assert alerts == []
+ state, alerts = cycle(state, held=set(), services=None)
+ assert titles(alerts) == [("critical", "mesh unreachable", "")]
+ state, alerts = cycle(state, held={"router-a"})
+ assert titles(alerts) == [("info", "mesh reachable again", "")]
+
+
+def test_no_router_and_no_control_plane_blames_the_cop():
+ state, _ = cycle(AgentMeshCopState())
+ for _ in range(3):
+ state, alerts = cycle(state, errors=["keys: unreachable"], listed=None, banned=None, held=set(),
+ services=None)
+ assert ("warning", "cop isolated", "") in titles(alerts)
+ assert all(a.title != "mesh unreachable" for a in alerts)
+
+
+def test_isolation_verdict_follows_a_later_control_plane_outage():
+ state, _ = cycle(AgentMeshCopState())
+ for _ in range(3):
+ state, alerts = cycle(state, held=set(), services=None)
+ assert titles(alerts) == [("critical", "mesh unreachable", "")]
+ for _ in range(3):
+ state, alerts = cycle(state, errors=["keys: unreachable"], listed=None, banned=None, held=set(),
+ services=None)
+ assert ("warning", "cop isolated", "") in titles(alerts)
+ state, alerts = cycle(state)
+ assert ("info", "cop reconnected", "") in titles(alerts)
+
+
+def test_isolation_alerts_before_any_router_list():
+ state = AgentMeshCopState()
+ for _ in range(3):
+ state, alerts = cycle(state, errors=["info: unreachable"], listed=None, banned=None, held=set(),
+ services=None)
+ assert ("warning", "cop isolated", "") in titles(alerts)
+
+
+def test_isolation_suppresses_per_router_and_service_alerts():
+ state, _ = cycle(AgentMeshCopState(), services={SERVICE_A})
+ for _ in range(4):
+ state, alerts = cycle(state, held=set(), services=set())
+ assert all(a.title not in ("router unreachable", "service disappeared") for a in alerts)
+
+
+@dataclass
+class FakeSync:
+ errors: List[str] = field(default_factory=list)
+ routers: Optional[list] = None
+ banned_peer_ids: Optional[list] = None
+
+
+class FakeSession:
+ """providers maps a service type to the peers the DHT names for it; catalogs maps a peer
+ to the (type number, name) pairs its list_local_services returns, or None when it does not answer."""
+
+ def __init__(self, sync_result, held, providers, router_addresses=(), keys=(b"\xaa" * 32,), connected=None,
+ catalogs=None):
+ self.sync_result = sync_result
+ self.catalogs = catalogs or {}
+ self.routers = [SimpleNamespace(peer_id=peer_id) for peer_id in held]
+ connected = held if connected is None else connected
+ self.host = SimpleNamespace(get_connected_peers=lambda: list(connected))
+ self.shunned_routers = {"router-z": 1e12}
+ self.providers = providers
+ self.mesh = SimpleNamespace(credential=SimpleNamespace(router_addresses=list(router_addresses),
+ control_plane_keys=list(keys)))
+ self.discovered = []
+
+ async def sync(self):
+ return self.sync_result
+
+ async def discover(self, service_type, limit):
+ self.discovered.append((service_type, limit))
+ return [SimpleNamespace(peer_id=peer_id) for peer_id in self.providers.get(service_type, [])]
+
+ async def call_tool(self, provider, target_service, tool, args):
+ assert (target_service, tool) == ("", "list_local_services")
+ catalog = self.catalogs.get(provider.peer_id)
+ if catalog is None:
+ raise ConnectionError("provider unreachable")
+ text = json.dumps([{"type": number, "name": name} for number, name in catalog])
+ return SimpleNamespace(is_error=False, text=[text])
+
+
+# take_snapshot drops sync entries that are not peer IDs, so its tests need real ones.
+ROUTER_PEER_ID = "12D3KooWPm5hFhFt3opdTK1TzfP1ofwnjXXqgHaRQU9dnUip7tDj"
+BANNED_PEER_ID = "12D3KooWMnTrGzU3jzwswYDmWdcmDxxZLi2bC5nYMzqHCZ3Hh1Kc"
+
+
+def test_take_snapshot_reads_sync_session_and_dht():
+ sync = FakeSync(routers=[SimpleNamespace(peer_id=ROUTER_PEER_ID)], banned_peer_ids=[BANNED_PEER_ID, "not-a-peer"])
+ session = FakeSession(sync, held=[ROUTER_PEER_ID], providers={"mcp": ["peer-1"], "inference": ["peer-2"]},
+ catalogs={"peer-1": [(1, "github")], "peer-2": [(2, "laptop-llm")]})
+ result = trio.run(take_snapshot, session)
+ assert result.listed_routers == {ROUTER_PEER_ID}
+ assert result.banned_peers == {BANNED_PEER_ID}
+ assert result.held_routers == {ROUTER_PEER_ID}
+ assert result.signing_keys == {"aa" * 32}
+ assert result.services == {SERVICE_A, SERVICE_B}
+ assert session.shunned_routers == {}
+
+
+def test_take_snapshot_sees_the_last_router_delisted_and_skips_dht_without_routers():
+ sync = FakeSync(routers=[], banned_peer_ids=[])
+ session = FakeSession(sync, held=[], providers={}, router_addresses=["/dns4/a/tcp/4001/p2p/router-a"])
+ result = trio.run(take_snapshot, session)
+ assert result.listed_routers == set()
+ assert result.services is None
+ assert session.discovered == []
+
+
+def test_take_snapshot_counts_only_connected_routers():
+ session = FakeSession(FakeSync(), held=["router-a", "router-b"], providers={}, connected=["router-a"])
+ assert trio.run(take_snapshot, session).held_routers == {"router-a"}
+
+
+def test_take_snapshot_marks_failed_info_as_unread():
+ session = FakeSession(FakeSync(errors=["info: timed out"]), held=["router-a"], providers={})
+ result = trio.run(take_snapshot, session)
+ assert result.listed_routers is None
+ assert result.banned_peers is None
+ assert result.sync_errors == ["info: timed out"]
+
+
+def test_fetch_services_asks_every_type_with_the_limit():
+ session = FakeSession(FakeSync(), held=["router-a"], providers={"mcp": ["peer-1"]},
+ catalogs={"peer-1": [(1, "github")]})
+ assert trio.run(fetch_services, session) == {SERVICE_A}
+ assert session.discovered == [("mcp", 200), ("inference", 200), ("a2a", 200), ("egress", 200)]
+
+
+def test_fetch_services_names_egress_destinations():
+ session = FakeSession(FakeSync(), held=["router-a"], providers={"egress": ["peer-4"]},
+ catalogs={"peer-4": [(4, "api.github.com")]})
+ assert trio.run(fetch_services, session) == {("egress", "api.github.com", "peer-4")}
+
+
+def test_fetch_services_keeps_only_advertised_types():
+ # peer-1 registered an a2a agent too, but advertises only mcp.
+ session = FakeSession(FakeSync(), held=["router-a"], providers={"mcp": ["peer-1"]},
+ catalogs={"peer-1": [(1, "github"), (3, "echo-agent")]})
+ assert trio.run(fetch_services, session) == {SERVICE_A}
+
+
+def test_fetch_services_leaves_out_a_provider_that_does_not_answer():
+ session = FakeSession(FakeSync(), held=["router-a"], providers={"mcp": ["peer-1", "peer-3"]},
+ catalogs={"peer-1": [(1, "github")], "peer-3": None})
+ assert trio.run(fetch_services, session) == {SERVICE_A}
+
+
+class RecordingChannel(Channel):
+ name = "recording"
+
+ def __init__(self):
+ self.messages = []
+
+ async def send(self, message):
+ self.messages.append(message)
+
+
+def test_enrollment_failure_alerts_once_and_is_retried():
+ attempts = []
+
+ def enroll():
+ attempts.append(1)
+ raise ConnectionError("control plane unreachable")
+
+ channel = RecordingChannel()
+
+ async def run_for_three_polls():
+ with trio.move_on_after(3 * CONFIG.poll_interval + 1):
+ await run_agentmesh_cop(enroll, CONFIG, [channel])
+
+ # A nonzero threshold lets the enrollment thread finish before the clock jumps.
+ trio.run(run_for_three_polls, clock=trio.testing.MockClock(autojump_threshold=0.05))
+ assert len(attempts) == 4
+ assert len(channel.messages) == 1
+ assert "cannot join the mesh" in channel.messages[0]