Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
feat(agentmesh-cop): watch the mesh as a member through the Python SDK
Tomas Tormo92f4d96d876f
diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.ymlindex e9993ddf..05659f01 100644--- a/.github/workflows/sdk.yml+++ b/.github/workflows/sdk.yml@@ -24,6 +24,7 @@ on:branches: [main]paths:- 'sdk/**'+ - 'development/tools/agentmesh-cop/**'- 'api/agentmesh.proto'- 'hack/gen-sdk-proto.sh'- 'hack/gen-sdk-docs/**'@@ -36,6 +37,7 @@ on:branches: [main]paths:- 'sdk/**'+ - 'development/tools/agentmesh-cop/**'- 'api/agentmesh.proto'- 'hack/gen-sdk-proto.sh'- 'hack/gen-sdk-docs/**'diff --git a/Makefile b/Makefileindex e3ea9e65..c1b0c5ec 100644--- a/Makefile+++ b/Makefile@@ -166,7 +166,8 @@ e2e-test: build docker-build# tests and leave it installed, which is what TestNativeSDKs needs to run# both against a real control plane instead of skipping. sdk-js also# compiles the example programs the docs embed; TestNativeSDKExamples runs-# them.+# them. sdk-python also runs agentmesh-cop's tests, since the cop runs on+# the Python SDK..PHONY: sdk-js sdk-python sdk-proto sdk-docs sdk-testsdk-js:cd sdk/js && npm ci --no-fund --no-audit && npm test && npm run build && npm run examples@@ -175,6 +176,7 @@ sdk-python:python3 -m venv sdk/python/.venv./sdk/python/.venv/bin/pip install -q -e './sdk/python[test,examples]'./sdk/python/.venv/bin/python -m pytest -q sdk/python/tests+ ./sdk/python/.venv/bin/python -m pytest -q development/tools/agentmesh-copsdk-proto:./hack/gen-sdk-proto.shdiff --git a/development/tools/agentmesh-cop/agentmesh_cop.py b/development/tools/agentmesh-cop/agentmesh_cop.pynew file mode 100644index 00000000..1af3ecff--- /dev/null+++ b/development/tools/agentmesh-cop/agentmesh_cop.py@@ -0,0 +1,403 @@+"""agentmesh-cop: a mesh member that watches the mesh and alerts on trouble.++It joins with the native Python SDK and, every poll, pulls the control plane+(signing keys, routers, bans), checks which listed routers admit it, and looks+the services up in the DHT. It holds a session with every router that admits+it and reserves no relay slot, since nothing needs to reach it.++Env: AGENTMESH_CONTROL_PLANE_URL; AGENTMESH_BOOTSTRAP_TOKEN_PATH or+AGENTMESH_JWT_PATH (first run only, see the SDK); AGENTMESH_STATE_DIR;+AGENTMESH_INSECURE_CONTROL_PLANE; POLL_INTERVAL, MISS_THRESHOLD; and the+channel settings in channels.py.+"""++import json+import logging+import os+import re+import sys+from dataclasses import dataclass, field+from datetime import datetime, timezone+from typing import Dict, List, Optional, Tuple++import trio+from agent_mesh.session import canonical_peer_ids++from channels import Channel, build_channels++logger = logging.getLogger("agentmesh-cop")+++@dataclass+class AgentMeshCopConfig:+ poll_interval: float+ miss_threshold: int+++def load_config(env) -> AgentMeshCopConfig:+ return AgentMeshCopConfig(+ poll_interval=float(env.get("POLL_INTERVAL", "30")),+ miss_threshold=int(env.get("MISS_THRESHOLD", "3")),+ )+++@dataclass+class Alert:+ severity: str+ title: str+ description: str+ peer_id: str = ""+ service: str = ""+++SEVERITY_EMOJI = {"critical": "🚨", "warning": "⚠️", "info": "ℹ️"}++CONTROL_CHARS_RE = re.compile(r"[\x00-\x1f\x7f]+")+++def sanitize_alert_text(text: str, limit: int = 200) -> str:+ """Strips control chars (incl. newlines) so attacker-controlled strings can't forge alert lines."""+ cleaned = CONTROL_CHARS_RE.sub(" ", text).strip()+ if len(cleaned) > limit:+ cleaned = cleaned[:limit].rstrip() + "..."+ return cleaned+++def format_alert(alert: Alert, cop_peer_id: str, timestamp: str) -> str:+ title = sanitize_alert_text(alert.title)+ lines = [f"{SEVERITY_EMOJI[alert.severity]} [{alert.severity.upper()}] {title}"]+ if alert.description:+ lines.append(sanitize_alert_text(alert.description))+ if alert.peer_id:+ lines.append(f"peer: {sanitize_alert_text(alert.peer_id)}")+ if alert.service:+ lines.append(f"service: {sanitize_alert_text(alert.service)}")+ lines.append(f"reported by {cop_peer_id} at {timestamp}")+ return "\n".join(lines)+++def delivery_error(error: Exception) -> str:+ # Channel URLs carry secrets (the Telegram bot token, the Slack webhook) and httpx errors quote the URL.+ response = getattr(error, "response", None)+ return f"HTTP {response.status_code}" if response is not None else type(error).__name__+++async def deliver(channels: List[Channel], message: str) -> None:+ async def deliver_to_channel(channel: Channel) -> None:+ for attempt in range(3):+ try:+ await channel.send(message)+ return+ except Exception as error:+ if attempt == 2:+ print(f"[-] {channel.name} delivery failed after 3 attempts: {delivery_error(error)}", flush=True)+ else:+ await trio.sleep(2**attempt)++ # Concurrent so one slow or failing channel's retries don't delay the others.+ async with trio.open_nursery() as nursery:+ for channel in channels:+ nursery.start_soon(deliver_to_channel, channel)+++@dataclass+class MeshSnapshot:+ """One poll's view of the mesh. None means that part could not be read this poll."""++ sync_errors: List[str]+ listed_routers: Optional[set]+ signing_keys: set+ banned_peers: Optional[set]+ held_routers: set+ services: Optional[set]+++@dataclass+class AgentMeshCopState:+ control_plane_misses: int = 0+ listed_routers: Optional[set] = None+ signing_keys: Optional[set] = None+ banned_peers: Optional[set] = None+ unreachable_polls: dict = field(default_factory=dict)+ unreachable_routers: set = field(default_factory=set)+ isolated_polls: int = 0+ isolation: Optional[str] = None # title of the isolation alert in force+ baseline: set = field(default_factory=set)+ miss_counts: dict = field(default_factory=dict)+ first_snapshot: bool = True+++def diff_alerts(previous: Optional[set], current: set, appeared: Alert, disappeared: Alert,+ describe=lambda item: item) -> List[Alert]:+ """One alert per item added to or dropped from a set; the first sighting is the baseline."""+ if previous is None:+ return []+ alerts = []+ for item in sorted(current - previous):+ alerts.append(Alert(appeared.severity, appeared.title, appeared.description, peer_id=describe(item)))+ for item in sorted(previous - current):+ alerts.append(Alert(disappeared.severity, disappeared.title, disappeared.description, peer_id=describe(item)))+ return alerts+++def detect_control_plane(state: AgentMeshCopState, snapshot: MeshSnapshot, miss_threshold: int) -> List[Alert]:+ alerts = []+ if snapshot.sync_errors:+ state.control_plane_misses += 1+ if state.control_plane_misses == miss_threshold:+ alerts.append(Alert("warning", "control plane sync failing",+ f"{miss_threshold} consecutive pulls failed: {'; '.join(snapshot.sync_errors)}"))+ else:+ if state.control_plane_misses >= miss_threshold:+ alerts.append(Alert("info", "control plane sync recovered",+ f"pull succeeded after {state.control_plane_misses} failures"))+ state.control_plane_misses = 0++ alerts.extend(diff_alerts(state.signing_keys, snapshot.signing_keys,+ Alert("critical", "new control-plane signing key", "a key the cop did not trust before"),+ Alert("warning", "control-plane signing key retired", "a trusted key left /keys"),+ describe=lambda key: f"key {key[:16]}..."))+ state.signing_keys = snapshot.signing_keys++ if snapshot.listed_routers is not None:+ alerts.extend(diff_alerts(state.listed_routers, snapshot.listed_routers,+ Alert("info", "router listed", "the control plane leased a new router"),+ Alert("warning", "router delisted", "the router's lease expired or was dropped")))+ state.listed_routers = snapshot.listed_routers++ if snapshot.banned_peers is not None:+ alerts.extend(diff_alerts(state.banned_peers, snapshot.banned_peers,+ Alert("critical", "peer banned", "the control plane banned this peer"),+ Alert("info", "peer unbanned", "the control plane lifted the ban")))+ state.banned_peers = snapshot.banned_peers+ return alerts+++ISOLATION_RECOVERY = {"cop isolated": "cop reconnected", "mesh unreachable": "mesh reachable again"}+++def detect_router_reachability(state: AgentMeshCopState, snapshot: MeshSnapshot, miss_threshold: int) -> List[Alert]:+ """Routers the control plane lists against the routers that admit the cop: a listed+ router that refuses a member refuses every new agent too."""+ # Before the first router list any held router counts, so a cop cut off from the start still alerts.+ held = snapshot.held_routers if state.listed_routers is None else snapshot.held_routers & state.listed_routers+ listed = state.listed_routers or set()+ alerts = []+ if not held:+ state.isolated_polls += 1+ if state.isolated_polls >= miss_threshold:+ if state.control_plane_misses >= miss_threshold:+ # Nothing answers the cop at all: its own network is the likelier fault.+ verdict = Alert("warning", "cop isolated",+ "neither the control plane nor any listed router answers this member")+ else:+ verdict = Alert("critical", "mesh unreachable",+ f"the control plane lists {len(listed)} router(s) and none admits a member")+ if verdict.title != state.isolation:+ alerts.append(verdict)+ state.isolation = verdict.title+ return alerts+ if state.isolation is not None:+ alerts.append(Alert("info", ISOLATION_RECOVERY[state.isolation],+ f"{len(held)} router(s) admit a member"))+ state.isolation = None+ state.isolated_polls = 0++ for router in sorted(listed - held):+ polls = state.unreachable_polls.get(router, 0) + 1+ state.unreachable_polls[router] = polls+ if polls == miss_threshold:+ state.unreachable_routers.add(router)+ alerts.append(Alert("warning", "router unreachable",+ f"listed by the control plane but refused or dropped a member for {polls} polls",+ peer_id=router))+ for router in sorted(state.unreachable_routers & held):+ alerts.append(Alert("info", "router reachable again", "the router admits a member again", peer_id=router))+ state.unreachable_routers &= listed - held+ state.unreachable_polls = {router: polls for router, polls in state.unreachable_polls.items()+ if router in listed - held}+ return alerts+++def detect_churn(state: AgentMeshCopState, services_snapshot: Optional[set], miss_threshold: int) -> List[Alert]:+ if services_snapshot is None or state.isolation is not None:+ return []+ if state.first_snapshot:+ state.baseline = set(services_snapshot)+ state.miss_counts = {}+ state.first_snapshot = False+ return []++ alerts = []+ for entry in sorted(services_snapshot - state.baseline):+ service_type, service_name, peer_id = entry+ alerts.append(Alert("info", "service appeared", f"{service_type}://{service_name} advertised on the mesh",+ peer_id=peer_id, service=f"{service_type}://{service_name}"))+ state.baseline.add(entry)+ for entry in list(state.miss_counts):+ if entry in services_snapshot:+ del state.miss_counts[entry]+ for entry in sorted(state.baseline - services_snapshot):+ misses = state.miss_counts.get(entry, 0) + 1+ if misses >= miss_threshold:+ service_type, service_name, peer_id = entry+ alerts.append(Alert("warning", "service disappeared",+ f"{service_type}://{service_name} missing for {misses} consecutive polls",+ peer_id=peer_id, service=f"{service_type}://{service_name}"))+ state.baseline.discard(entry)+ state.miss_counts.pop(entry, None)+ else:+ state.miss_counts[entry] = misses+ return alerts+++def evaluate_cycle(state: AgentMeshCopState, snapshot: MeshSnapshot, config: AgentMeshCopConfig):+ alerts = []+ alerts.extend(detect_control_plane(state, snapshot, config.miss_threshold))+ alerts.extend(detect_router_reachability(state, snapshot, config.miss_threshold))+ alerts.extend(detect_churn(state, snapshot.services, config.miss_threshold))+ return state, alerts+++SERVICE_TYPES = ["mcp", "inference", "a2a", "egress"]+DISCOVERY_LIMIT = 200+# ServiceType numbers in api/agentmesh.proto, as list_local_services renders them.+SERVICE_TYPE_NAMES = {1: "mcp", 2: "inference", 3: "a2a", 4: "egress"}+PROVIDER_QUERY_TIMEOUT = 10.0+++async def provider_services(session, provider) -> List[Tuple[str, str]]:+ """The (type, name) pairs a provider registered, from its catalog's list_local_services."""+ result = await session.call_tool(provider, "", "list_local_services", {})+ if result.is_error or not result.text:+ raise RuntimeError(f"list_local_services failed: {result.text}")+ return [(SERVICE_TYPE_NAMES.get(service.get("type")), service.get("name", ""))+ for service in json.loads(result.text[0])]+++async def fetch_services(session) -> set:+ # The DHT names the providers of a type, not their services; each provider names its own.+ providers: Dict[str, object] = {}+ advertised_types: Dict[str, set] = {}+ for service_type in SERVICE_TYPES:+ for provider in await session.discover(service_type, limit=DISCOVERY_LIMIT):+ providers[provider.peer_id] = provider+ advertised_types.setdefault(provider.peer_id, set()).add(service_type)+ snapshot = set()+ for peer_id, provider in providers.items():+ try:+ with trio.fail_after(PROVIDER_QUERY_TIMEOUT):+ services = await provider_services(session, provider)+ except Exception as error:+ # Left out of this poll: its services count as missed, so only a lasting silence alerts.+ # Debug only, since a dead provider's DHT record keeps it here for hours.+ logger.debug("provider %s did not list its services: %s", peer_id, error)+ continue+ # A node lists every service it registered, advertised in the DHT or not.+ snapshot.update((service_type, name, peer_id) for service_type, name in services+ if service_type in advertised_types[peer_id])+ return snapshot+++async def take_snapshot(session) -> MeshSnapshot:+ # The SDK keeps away from a failed router for minutes; the cop probes every listed router each poll.+ session.shunned_routers.clear()+ result = await session.sync()+ # Raw wire strings; the session keys its routers on the canonical form, so compare on that.+ listed_routers = (set(canonical_peer_ids([router.peer_id for router in result.routers]))+ if result.routers is not None else None)+ banned_peers = set(canonical_peer_ids(result.banned_peer_ids)) if result.banned_peer_ids is not None else None+ # session.routers keeps a dead router until its redials run out; count only live connections.+ connected = {str(peer) for peer in session.host.get_connected_peers()}+ held_routers = {router.peer_id for router in session.routers if router.peer_id in connected}+ services = None+ if held_routers:+ try:+ services = await fetch_services(session)+ except Exception as error:+ print(f"[-] service discovery failed: {error}", flush=True)+ return MeshSnapshot(+ sync_errors=list(result.errors),+ listed_routers=listed_routers,+ signing_keys={bytes(key).hex() for key in session.mesh.credential.control_plane_keys},+ banned_peers=banned_peers,+ held_routers=held_routers,+ services=services,+ )+++async def alert_all(channels: List[Channel], alerts: List[Alert], cop_peer_id: str) -> None:+ timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds")+ for alert in alerts:+ await deliver(channels, format_alert(alert, cop_peer_id, timestamp))+++# More routers than any mesh lists: the session then holds every router that admits it,+# and its own loop redials the ones that drop and retries the ones it shunned.+HOLD_EVERY_ROUTER = 10_000+++async def watch(session, channels: List[Channel], config: AgentMeshCopConfig) -> None:+ state = AgentMeshCopState()+ while True:+ try:+ state, alerts = evaluate_cycle(state, await take_snapshot(session), config)+ await alert_all(channels, alerts, session.peer_id)+ except Exception as error:+ print(f"[-] poll cycle failed: {error}", flush=True)+ await trio.sleep(config.poll_interval)+++async def run_agentmesh_cop(enroll, config: AgentMeshCopConfig, channels: List[Channel]) -> None:+ mesh = None+ join_failed = False+ while True:+ try:+ if mesh is None:+ # Blocks on HTTP and, without auto-approve, on an operator, so it runs off the trio thread.+ mesh = await trio.to_thread.run_sync(enroll)+ # The cop drives every pull itself; key rotation events still bring one forward.+ async with mesh.join(routers=HOLD_EVERY_ROUTER, reserve=False, control_plane_sync_interval=0) as session:+ print(f"[+] on the mesh as {session.peer_id}, holding {len(session.routers)} router(s)", flush=True)+ if join_failed:+ await alert_all(channels, [Alert("info", "joined the mesh again", "")], session.peer_id)+ join_failed = False+ await watch(session, channels, config)+ except Exception as error:+ print(f"[-] cannot join the mesh: {error}", flush=True)+ if not join_failed:+ reporter = mesh.peer_id if mesh is not None else "a cop that is not enrolled yet"+ await alert_all(channels, [Alert("critical", "cannot join the mesh", str(error))], reporter)+ join_failed = True+ await trio.sleep(config.poll_interval)+++def main() -> None:+ from agent_mesh import AgentMesh++ # Logs every refused dial, some with a full traceback; the cop reports reachability itself.+ logging.getLogger("libp2p.transport").setLevel(logging.CRITICAL)+ config = load_config(os.environ)+ channels = build_channels(os.environ)+ channel_names = ", ".join(channel.name for channel in channels)+ print(f"[*] agentmesh-cop starting: poll={config.poll_interval}s miss_threshold={config.miss_threshold} "+ f"channels=[{channel_names}]", flush=True)+ control_plane_url = os.environ["AGENTMESH_CONTROL_PLANE_URL"]++ def enroll():+ return AgentMesh.enroll(+ control_plane_url,+ bootstrap_token_path=os.environ.get("AGENTMESH_BOOTSTRAP_TOKEN_PATH"),+ jwt_path=os.environ.get("AGENTMESH_JWT_PATH"),+ state_dir=os.environ.get("AGENTMESH_STATE_DIR", "~/.config/agentmesh/cop"),+ allow_insecure=os.environ.get("AGENTMESH_INSECURE_CONTROL_PLANE") == "true",+ )++ trio.run(run_agentmesh_cop, enroll, config, channels)+++if __name__ == "__main__":+ try:+ main()+ except KeyboardInterrupt:+ sys.exit(0)diff --git a/development/tools/agentmesh-cop/channels.py b/development/tools/agentmesh-cop/channels.pynew file mode 100644index 00000000..78e576ff--- /dev/null+++ b/development/tools/agentmesh-cop/channels.py@@ -0,0 +1,67 @@+import abc+from typing import List++import httpx+++class Channel(abc.ABC):+ """Delivery backend. Subclasses self-register; declare required_env and build_channels() picks them up."""++ name = "channel"+ required_env: tuple = ()+ registry: list = []++ def __init_subclass__(cls, **kwargs):+ super().__init_subclass__(**kwargs)+ Channel.registry.append(cls)++ @classmethod+ def from_env(cls, env):+ # __init__ parameters must line up with required_env order.+ return cls(*(env[var] for var in cls.required_env))++ @abc.abstractmethod+ async def send(self, message: str) -> None: ...+++class SlackChannel(Channel):+ name = "slack"+ required_env = ("SLACK_WEBHOOK_URL",)++ def __init__(self, webhook_url: str):+ self.webhook_url = webhook_url++ async def send(self, message: str) -> None:+ async with httpx.AsyncClient() as http_client:+ response = await http_client.post(self.webhook_url, json={"text": message}, timeout=10.0)+ response.raise_for_status()+++class TelegramChannel(Channel):+ name = "telegram"+ required_env = ("TELEGRAM_BOT_TOKEN", "TELEGRAM_CHAT_ID")++ def __init__(self, bot_token: str, chat_id: str):+ self.bot_token = bot_token+ self.chat_id = chat_id++ async def send(self, message: str) -> None:+ url = f"https://api.telegram.org/bot{self.bot_token}/sendMessage"+ async with httpx.AsyncClient() as http_client:+ response = await http_client.post(url, json={"chat_id": self.chat_id, "text": message}, timeout=10.0)+ response.raise_for_status()+++class StdoutChannel(Channel):+ name = "stdout"++ async def send(self, message: str) -> None:+ print(f"[ALERT] {message}", flush=True)+++def build_channels(env) -> List[Channel]:+ # Stdout is always on so the log shows every alert, delivered or not.+ channels = [cls.from_env(env) for cls in Channel.registry+ if cls.required_env and all(env.get(var) for var in cls.required_env)]+ channels.append(StdoutChannel())+ return channelsdiff --git a/development/tools/agentmesh-cop/test_agentmesh_cop.py b/development/tools/agentmesh-cop/test_agentmesh_cop.pynew file mode 100644index 00000000..63bf9867--- /dev/null+++ b/development/tools/agentmesh-cop/test_agentmesh_cop.py@@ -0,0 +1,444 @@+import json+from dataclasses import dataclass, field+from types import SimpleNamespace+from typing import List, Optional++import httpx+import trio+import trio.testing++from agentmesh_cop import (+ AgentMeshCopConfig,+ AgentMeshCopState,+ Alert,+ MeshSnapshot,+ deliver,+ evaluate_cycle,+ fetch_services,+ format_alert,+ load_config,+ run_agentmesh_cop,+ sanitize_alert_text,+ take_snapshot,+)+from channels import (+ Channel,+ SlackChannel,+ StdoutChannel,+ TelegramChannel,+ build_channels,+)+++def test_load_config_defaults():+ config = load_config({})+ assert config.poll_interval == 30.0+ assert config.miss_threshold == 3+++def test_load_config_overrides():+ config = load_config({"POLL_INTERVAL": "5", "MISS_THRESHOLD": "2"})+ assert config.poll_interval == 5.0+ assert config.miss_threshold == 2+++def test_build_channels_stdout_fallback():+ channels = build_channels({})+ assert len(channels) == 1+ assert isinstance(channels[0], StdoutChannel)+++def test_build_channels_slack_and_telegram_keep_stdout():+ env = {+ "SLACK_WEBHOOK_URL": "https://hooks.slack.example/x",+ "TELEGRAM_BOT_TOKEN": "token",+ "TELEGRAM_CHAT_ID": "42",+ }+ channels = build_channels(env)+ assert {type(c) for c in channels} == {SlackChannel, TelegramChannel, StdoutChannel}+++def test_build_channels_telegram_requires_chat_id():+ channels = build_channels({"TELEGRAM_BOT_TOKEN": "token"})+ assert len(channels) == 1+ assert isinstance(channels[0], StdoutChannel)+++def test_build_channels_picks_up_registered_subclass():+ class EchoChannel(Channel):+ name = "echo"+ required_env = ("ECHO_TARGET",)++ def __init__(self, target):+ self.target = target++ async def send(self, message):+ pass++ try:+ channels = build_channels({"ECHO_TARGET": "x"})+ assert [type(c) for c in channels] == [EchoChannel, StdoutChannel]+ assert channels[0].target == "x"+ finally:+ Channel.registry.remove(EchoChannel)+++def test_format_alert():+ alert = Alert("critical", "peer banned", "the control plane banned this peer", peer_id="12D3KooPeer")+ message = format_alert(alert, "12D3KooCop", "2026-08-06T10:00:00+00:00")+ assert message.startswith("🚨 [CRITICAL] peer banned")+ assert "the control plane banned this peer" in message+ assert "peer: 12D3KooPeer" in message+ assert "reported by 12D3KooCop at 2026-08-06T10:00:00+00:00" in message+++class FlakyChannel(Channel):+ name = "flaky"++ def __init__(self, failures):+ self.failures = failures+ self.attempts = 0+ self.delivered = []++ async def send(self, message):+ self.attempts += 1+ if self.attempts <= self.failures:+ raise RuntimeError("boom")+ self.delivered.append(message)+++def test_deliver_retries_then_succeeds():+ channel = FlakyChannel(failures=2)+ trio.run(deliver, [channel], "hello")+ assert channel.delivered == ["hello"]+ assert channel.attempts == 3+++def test_deliver_failure_does_not_log_the_channel_secret(capsys):+ class RejectingChannel(Channel):+ name = "telegram"++ async def send(self, message):+ request = httpx.Request("POST", "https://api.telegram.org/botSECRET-TOKEN/sendMessage")+ response = httpx.Response(401, request=request)+ response.raise_for_status()++ trio.run(deliver, [RejectingChannel()], "hello")+ output = capsys.readouterr().out+ assert "HTTP 401" in output+ assert "SECRET-TOKEN" not in output+++def test_deliver_gives_up_without_raising():+ channel = FlakyChannel(failures=5)+ trio.run(deliver, [channel], "hello")+ assert channel.delivered == []+ assert channel.attempts == 3+++def test_sanitize_alert_text_strips_control_chars_and_collapses():+ hostile = "svc\n🚨 [CRITICAL] fake alert\x07more"+ sanitized = sanitize_alert_text(hostile)+ assert "\n" not in sanitized+ assert "\x07" not in sanitized+ assert "svc" in sanitized and "fake alert" in sanitized+++def test_sanitize_alert_text_truncates_with_ellipsis():+ truncated = sanitize_alert_text("a" * 500, limit=50)+ assert len(truncated) == 53+ assert truncated.endswith("...")+++def test_format_alert_sanitizes_hostile_service_name():+ hostile_service = "svc\n🚨 [CRITICAL] fake peer banned"+ alert = Alert("info", "service appeared", "desc", peer_id="peer-1", service=hostile_service)+ message = format_alert(alert, "12D3KooCop", "2026-08-06T10:00:00+00:00")+ lines = message.split("\n")+ assert len(lines) == 5 # no forged extra line from the injected newline+ assert lines[0] == "ℹ️ [INFO] service appeared"+ assert "svc" in lines[3] and "CRITICAL" in lines[3]+++CONFIG = AgentMeshCopConfig(poll_interval=30.0, miss_threshold=3)+ROUTERS = {"router-a", "router-b"}+KEYS = {"aa" * 32}+SERVICE_A = ("mcp", "github", "peer-1")+SERVICE_B = ("inference", "laptop-llm", "peer-2")+++def snapshot(errors=(), listed=ROUTERS, keys=KEYS, banned=frozenset(), held=ROUTERS, services=frozenset()):+ return MeshSnapshot(+ sync_errors=list(errors),+ listed_routers=None if listed is None else set(listed),+ signing_keys=set(keys),+ banned_peers=None if banned is None else set(banned),+ held_routers=set(held),+ services=None if services is None else set(services),+ )+++def cycle(state, **kwargs):+ return evaluate_cycle(state, snapshot(**kwargs), CONFIG)+++def titles(alerts):+ return [(a.severity, a.title, a.peer_id) for a in alerts]+++def test_first_poll_is_a_baseline():+ _, alerts = cycle(AgentMeshCopState(), services={SERVICE_A})+ assert alerts == []+++def test_service_provider_appears_and_disappears_after_misses():+ state, _ = cycle(AgentMeshCopState(), services={SERVICE_A})+ state, alerts = cycle(state, services={SERVICE_A, SERVICE_B})+ assert titles(alerts) == [("info", "service appeared", "peer-2")]+ assert alerts[0].service == "inference://laptop-llm"+ for _ in range(2):+ state, alerts = cycle(state, services={SERVICE_B})+ assert alerts == []+ state, alerts = cycle(state, services={SERVICE_B})+ assert titles(alerts) == [("warning", "service disappeared", "peer-1")]+ assert alerts[0].service == "mcp://github"+++def test_signing_key_added_and_retired():+ state, _ = cycle(AgentMeshCopState())+ state, alerts = cycle(state, keys=KEYS | {"bb" * 32})+ assert titles(alerts) == [("critical", "new control-plane signing key", "key bbbbbbbbbbbbbbbb...")]+ state, alerts = cycle(state, keys={"bb" * 32})+ assert titles(alerts) == [("warning", "control-plane signing key retired", "key aaaaaaaaaaaaaaaa...")]+++def test_router_listed_and_delisted():+ state, _ = cycle(AgentMeshCopState())+ state, alerts = cycle(state, listed={"router-a", "router-c"}, held={"router-a", "router-c"})+ assert titles(alerts) == [("info", "router listed", "router-c"), ("warning", "router delisted", "router-b")]+++def test_failed_info_keeps_the_last_router_list():+ state, _ = cycle(AgentMeshCopState())+ state, alerts = cycle(state, listed=None, banned=None)+ assert alerts == []+ assert state.listed_routers == ROUTERS+++def test_peer_banned_and_unbanned():+ state, _ = cycle(AgentMeshCopState())+ state, alerts = cycle(state, banned={"peer-x"})+ assert titles(alerts) == [("critical", "peer banned", "peer-x")]+ state, alerts = cycle(state, banned=set())+ assert titles(alerts) == [("info", "peer unbanned", "peer-x")]+++def test_control_plane_failing_alerts_once_at_threshold_then_recovers():+ state, _ = cycle(AgentMeshCopState())+ for _ in range(2):+ state, alerts = cycle(state, errors=["info: timed out"], listed=None, banned=None)+ assert alerts == []+ state, alerts = cycle(state, errors=["info: timed out"], listed=None, banned=None)+ assert titles(alerts) == [("warning", "control plane sync failing", "")]+ assert "info: timed out" in alerts[0].description+ state, alerts = cycle(state, errors=["info: timed out"], listed=None, banned=None)+ assert alerts == []+ state, alerts = cycle(state)+ assert titles(alerts) == [("info", "control plane sync recovered", "")]+++def test_listed_router_that_refuses_a_member_alerts_after_misses_then_recovers():+ state, _ = cycle(AgentMeshCopState())+ for _ in range(2):+ state, alerts = cycle(state, held={"router-a"})+ assert alerts == []+ state, alerts = cycle(state, held={"router-a"})+ assert titles(alerts) == [("warning", "router unreachable", "router-b")]+ state, alerts = cycle(state, held={"router-a"})+ assert alerts == []+ state, alerts = cycle(state)+ assert titles(alerts) == [("info", "router reachable again", "router-b")]+++def test_no_router_admits_while_control_plane_answers_is_a_mesh_outage():+ state, _ = cycle(AgentMeshCopState())+ for _ in range(2):+ state, alerts = cycle(state, held=set(), services=None)+ assert alerts == []+ state, alerts = cycle(state, held=set(), services=None)+ assert titles(alerts) == [("critical", "mesh unreachable", "")]+ state, alerts = cycle(state, held={"router-a"})+ assert titles(alerts) == [("info", "mesh reachable again", "")]+++def test_no_router_and_no_control_plane_blames_the_cop():+ state, _ = cycle(AgentMeshCopState())+ for _ in range(3):+ state, alerts = cycle(state, errors=["keys: unreachable"], listed=None, banned=None, held=set(),+ services=None)+ assert ("warning", "cop isolated", "") in titles(alerts)+ assert all(a.title != "mesh unreachable" for a in alerts)+++def test_isolation_verdict_follows_a_later_control_plane_outage():+ state, _ = cycle(AgentMeshCopState())+ for _ in range(3):+ state, alerts = cycle(state, held=set(), services=None)+ assert titles(alerts) == [("critical", "mesh unreachable", "")]+ for _ in range(3):+ state, alerts = cycle(state, errors=["keys: unreachable"], listed=None, banned=None, held=set(),+ services=None)+ assert ("warning", "cop isolated", "") in titles(alerts)+ state, alerts = cycle(state)+ assert ("info", "cop reconnected", "") in titles(alerts)+++def test_isolation_alerts_before_any_router_list():+ state = AgentMeshCopState()+ for _ in range(3):+ state, alerts = cycle(state, errors=["info: unreachable"], listed=None, banned=None, held=set(),+ services=None)+ assert ("warning", "cop isolated", "") in titles(alerts)+++def test_isolation_suppresses_per_router_and_service_alerts():+ state, _ = cycle(AgentMeshCopState(), services={SERVICE_A})+ for _ in range(4):+ state, alerts = cycle(state, held=set(), services=set())+ assert all(a.title not in ("router unreachable", "service disappeared") for a in alerts)+++@dataclass+class FakeSync:+ errors: List[str] = field(default_factory=list)+ routers: Optional[list] = None+ banned_peer_ids: Optional[list] = None+++class FakeSession:+ """providers maps a service type to the peers the DHT names for it; catalogs maps a peer+ to the (type number, name) pairs its list_local_services returns, or None when it does not answer."""++ def __init__(self, sync_result, held, providers, router_addresses=(), keys=(b"\xaa" * 32,), connected=None,+ catalogs=None):+ self.sync_result = sync_result+ self.catalogs = catalogs or {}+ self.routers = [SimpleNamespace(peer_id=peer_id) for peer_id in held]+ connected = held if connected is None else connected+ self.host = SimpleNamespace(get_connected_peers=lambda: list(connected))+ self.shunned_routers = {"router-z": 1e12}+ self.providers = providers+ self.mesh = SimpleNamespace(credential=SimpleNamespace(router_addresses=list(router_addresses),+ control_plane_keys=list(keys)))+ self.discovered = []++ async def sync(self):+ return self.sync_result++ async def discover(self, service_type, limit):+ self.discovered.append((service_type, limit))+ return [SimpleNamespace(peer_id=peer_id) for peer_id in self.providers.get(service_type, [])]++ async def call_tool(self, provider, target_service, tool, args):+ assert (target_service, tool) == ("", "list_local_services")+ catalog = self.catalogs.get(provider.peer_id)+ if catalog is None:+ raise ConnectionError("provider unreachable")+ text = json.dumps([{"type": number, "name": name} for number, name in catalog])+ return SimpleNamespace(is_error=False, text=[text])+++# take_snapshot drops sync entries that are not peer IDs, so its tests need real ones.+ROUTER_PEER_ID = "12D3KooWPm5hFhFt3opdTK1TzfP1ofwnjXXqgHaRQU9dnUip7tDj"+BANNED_PEER_ID = "12D3KooWMnTrGzU3jzwswYDmWdcmDxxZLi2bC5nYMzqHCZ3Hh1Kc"+++def test_take_snapshot_reads_sync_session_and_dht():+ sync = FakeSync(routers=[SimpleNamespace(peer_id=ROUTER_PEER_ID)], banned_peer_ids=[BANNED_PEER_ID, "not-a-peer"])+ session = FakeSession(sync, held=[ROUTER_PEER_ID], providers={"mcp": ["peer-1"], "inference": ["peer-2"]},+ catalogs={"peer-1": [(1, "github")], "peer-2": [(2, "laptop-llm")]})+ result = trio.run(take_snapshot, session)+ assert result.listed_routers == {ROUTER_PEER_ID}+ assert result.banned_peers == {BANNED_PEER_ID}+ assert result.held_routers == {ROUTER_PEER_ID}+ assert result.signing_keys == {"aa" * 32}+ assert result.services == {SERVICE_A, SERVICE_B}+ assert session.shunned_routers == {}+++def test_take_snapshot_sees_the_last_router_delisted_and_skips_dht_without_routers():+ sync = FakeSync(routers=[], banned_peer_ids=[])+ session = FakeSession(sync, held=[], providers={}, router_addresses=["/dns4/a/tcp/4001/p2p/router-a"])+ result = trio.run(take_snapshot, session)+ assert result.listed_routers == set()+ assert result.services is None+ assert session.discovered == []+++def test_take_snapshot_counts_only_connected_routers():+ session = FakeSession(FakeSync(), held=["router-a", "router-b"], providers={}, connected=["router-a"])+ assert trio.run(take_snapshot, session).held_routers == {"router-a"}+++def test_take_snapshot_marks_failed_info_as_unread():+ session = FakeSession(FakeSync(errors=["info: timed out"]), held=["router-a"], providers={})+ result = trio.run(take_snapshot, session)+ assert result.listed_routers is None+ assert result.banned_peers is None+ assert result.sync_errors == ["info: timed out"]+++def test_fetch_services_asks_every_type_with_the_limit():+ session = FakeSession(FakeSync(), held=["router-a"], providers={"mcp": ["peer-1"]},+ catalogs={"peer-1": [(1, "github")]})+ assert trio.run(fetch_services, session) == {SERVICE_A}+ assert session.discovered == [("mcp", 200), ("inference", 200), ("a2a", 200), ("egress", 200)]+++def test_fetch_services_names_egress_destinations():+ session = FakeSession(FakeSync(), held=["router-a"], providers={"egress": ["peer-4"]},+ catalogs={"peer-4": [(4, "api.github.com")]})+ assert trio.run(fetch_services, session) == {("egress", "api.github.com", "peer-4")}+++def test_fetch_services_keeps_only_advertised_types():+ # peer-1 registered an a2a agent too, but advertises only mcp.+ session = FakeSession(FakeSync(), held=["router-a"], providers={"mcp": ["peer-1"]},+ catalogs={"peer-1": [(1, "github"), (3, "echo-agent")]})+ assert trio.run(fetch_services, session) == {SERVICE_A}+++def test_fetch_services_leaves_out_a_provider_that_does_not_answer():+ session = FakeSession(FakeSync(), held=["router-a"], providers={"mcp": ["peer-1", "peer-3"]},+ catalogs={"peer-1": [(1, "github")], "peer-3": None})+ assert trio.run(fetch_services, session) == {SERVICE_A}+++class RecordingChannel(Channel):+ name = "recording"++ def __init__(self):+ self.messages = []++ async def send(self, message):+ self.messages.append(message)+++def test_enrollment_failure_alerts_once_and_is_retried():+ attempts = []++ def enroll():+ attempts.append(1)+ raise ConnectionError("control plane unreachable")++ channel = RecordingChannel()++ async def run_for_three_polls():+ with trio.move_on_after(3 * CONFIG.poll_interval + 1):+ await run_agentmesh_cop(enroll, CONFIG, [channel])++ # A nonzero threshold lets the enrollment thread finish before the clock jumps.+ trio.run(run_for_three_polls, clock=trio.testing.MockClock(autojump_threshold=0.05))+ assert len(attempts) == 4+ assert len(channel.messages) == 1+ assert "cannot join the mesh" in channel.messages[0]