Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
e2e: watch the mesh, not the containers' logs
Antonio Ojeac63b27f04cc3
diff --git a/tests/e2e/a2a_mesh.bats b/tests/e2e/a2a_mesh.batsindex 402535cb..313e9aad 100644--- a/tests/e2e/a2a_mesh.bats+++ b/tests/e2e/a2a_mesh.bats@@ -20,7 +20,7 @@ start_a2a_echo() {--network-alias a2a-echo \"${A2A_ECHO_IMAGE}" >/dev/nullMESH_CONTAINERS+=("${name}")- mesh_wait_for_log "${name}" "Uvicorn running on" 30+ mesh_wait_for_http "http://a2a-echo:7777/" 30}setup() {@@ -43,8 +43,7 @@ teardown() {echo "[$(date +%T)] Starting Node 1 (consumer)"mesh_start_node 1 "--log-level debug"- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60- mesh_wait_for_mcp_ready 1 20+ mesh_wait_for_mcp_ready 1 60echo "[$(date +%T)] Starting a2a echo agent backend"start_a2a_echo@@ -53,15 +52,14 @@ teardown() {mesh_start_node 2 \"--log-level debug" \"tests/e2e/docker/a2a-echo/sam-node-config.yaml"- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20- mesh_wait_for_mcp_ready 2 20+ mesh_wait_for_mcp_ready 2 30local node2_peer_id- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')+ node2_peer_id=$(mesh_node_peer_id 2)+ [[ -n "${node2_peer_id}" ]]echo "[$(date +%T)] Connecting Node 1 to Node 2"- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"- run mesh_connect_peer 1 "${node2_addr}"+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"[[ "$status" -eq 0 ]]mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20diff --git a/tests/e2e/agent_sandbox.bats b/tests/e2e/agent_sandbox.batsindex 1d394701..12ad0c9d 100644--- a/tests/e2e/agent_sandbox.bats+++ b/tests/e2e/agent_sandbox.bats@@ -37,8 +37,7 @@ agent_curl() {# HOME is unwritable and the default location cannot be created.mesh_start_node 1 "--socket-path /sockets/node.sock --data-dir /sockets/node-data" "" \"-v ${MESH_SOCKET_DIR}:/sockets --user $(id -u):$(id -g) -e HOME=/sockets"- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60- mesh_wait_for_mcp_ready 1 20+ mesh_wait_for_mcp_ready 1 60# The gateway consumes the node. It has no network either: the node's socket# is the only thing it needs.diff --git a/tests/e2e/auth_flows.bats b/tests/e2e/auth_flows.batsindex 145c37dd..80e58674 100644--- a/tests/e2e/auth_flows.bats+++ b/tests/e2e/auth_flows.bats@@ -10,6 +10,24 @@ teardown() {mesh_cleanup_env}+# assert_enrolled fails unless the control plane holds an enrollment for the+# node named <idx>, made the given way; it prints the record.+assert_enrolled() {+ local idx="$1"+ local enrollment_type="$2"+ local peer_id record+ peer_id="$(mesh_node_peer_id "${idx}")"+ [[ -n "${peer_id}" ]]+ record="$(mesh_enrolled_node "${peer_id}")"+ if [[ -z "${record}" ]]; then+ echo "control plane has no enrollment for ${peer_id}" >&2+ return 1+ fi+ [[ "$(jq -r '.EnrollmentType' <<<"${record}")" == "${enrollment_type}" ]]+ [[ "$(jq -r '.Role' <<<"${record}")" == "sam:role:node" ]]+ echo "${record}"+}+@test "Authentication Flow 1: Client Credentials Flow" {run mesh_start_mock_oidc[[ "$status" -eq 0 ]]@@ -20,8 +38,8 @@ teardown() {# mesh_start_node uses --token-url by default, which implements Client Credentials flowmesh_start_node 1- run mesh_assert_container_running "${MESH_PREFIX}-node-1"- [[ "$status" -eq 0 ]]+ mesh_wait_for_mcp_ready 1 30+ assert_enrolled 1 OIDC}@test "Authentication Flow 2: Device Authorization Flow (Interactive)" {@@ -31,16 +49,12 @@ teardown() {run mesh_start_router[[ "$status" -eq 0 ]]- # 1. Get a token from mock provider using Python helper- local token- token=$(docker run --rm --network "${MESH_NETWORK}" $(mesh_get_add_hosts) python:3.12 python3 -c "import urllib.request; import json; req = urllib.request.Request('http://mock-oidc:18080/token', data=b''); resp = urllib.request.urlopen(req); print(json.loads(resp.read().decode())['access_token'])")-- [[ -n "${token}" ]]-- # 2. Run sam-node join to enroll and store identity. The sam-node image has+ # Run sam-node join to enroll and store identity. The sam-node image has# no browser to open, so the loopback flow can't complete; join falls back# to the OAuth 2.0 Device Authorization Grant (RFC 8628), which the mock- # OIDC provider advertises and auto-approves on the first poll.+ # OIDC provider advertises and auto-approves on the first poll. The+ # provider serves the device grant only for the code it handed out, so an+ # enrollment is proof the device flow ran.local node_name="${MESH_PREFIX}-node-login"local router_peer_idrouter_peer_id=$(cat "/tmp/${MESH_PREFIX}-router-peer-id")@@ -63,15 +77,11 @@ teardown() {join --config /etc/sam/node-config.yaml --data-dir /data --insecure-control-plane "http://sam-control-plane:8080"MESH_CONTAINERS+=("${node_name}-join")- run mesh_wait_for_log "${node_name}-join" "OAuth Device Authorization Flow" 20- [[ "$status" -eq 0 ]]- run mesh_wait_for_log "${node_name}-join" "Enter code: ABCD-1234" 20- [[ "$status" -eq 0 ]]-- # Wait for the join process to finish and check it succeeded.- run mesh_wait_for_log "${node_name}-join" "Successfully joined the Sovereign Agent Mesh!" 20- [[ "$status" -eq 0 ]]- [[ "$(docker inspect -f '{{.State.ExitCode}}' "${node_name}-join")" -eq 0 ]]+ # join is a command: it ends, and its exit code is its verdict.+ run timeout 60 docker wait "${node_name}-join"+ echo "join exit: $output"+ [[ "$status" -eq 0 && "$output" == "0" ]] || mesh_container_log "${node_name}-join"+ [[ "$status" -eq 0 && "$output" == "0" ]]docker rm -f "${node_name}-join" >/dev/null 2>&1 || true# Now run the node with the stored identity, on the same control plane it@@ -89,8 +99,13 @@ teardown() {--insecure-control-planeMESH_CONTAINERS+=("${node_name}")- mesh_wait_for_log "${node_name}" "Using stored identity." 20- mesh_wait_for_log "${node_name}" "Serving the local API on Unix socket" 30+ mesh_wait_for_mcp_ready login 30++ # Enrolled by the user the provider signed in, through the device grant.+ local record+ record="$(assert_enrolled login OIDC)"+ echo "enrollment: ${record}"+ [[ "$(jq -r '.ClaimsJSON | fromjson | .sub' <<<"${record}")" == "test-user" ]]# The labels join declared must come back attested. /sam/identity hands back# the raw biscuit rather than decoded claims, so read the signed@@ -123,9 +138,12 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in rawrun mesh_start_router[[ "$status" -eq 0 ]]- # 1. Get a token from mock provider+ # 1. Get a token from the mock provider the way a workload platform would+ # mint one: a real grant, not an empty POST.local token- token=$(docker run --rm --network "${MESH_NETWORK}" $(mesh_get_add_hosts) python:3.12 python3 -c "import urllib.request; import json; req = urllib.request.Request('http://mock-oidc:18080/token', data=b''); resp = urllib.request.urlopen(req); print(json.loads(resp.read().decode())['access_token'])")+ token=$(docker run --rm --network "${MESH_NETWORK}" $(mesh_get_add_hosts) "${MESH_RUNTIME_IMAGE}" \+ curl -sf --max-time 10 -X POST -d 'grant_type=client_credentials&client_id=sam-mesh-audience' \+ http://mock-oidc:18080/token | jq -r '.access_token // empty')[[ -n "${token}" ]]@@ -157,7 +175,8 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in raw--jwt-path "/var/run/secrets/tokens/sa-token"MESH_CONTAINERS+=("${node_name}")- mesh_wait_for_log "${node_name}" "SAM Node Online" 20+ mesh_wait_for_mcp_ready wi 30+ assert_enrolled wi OIDC}@test "Authentication Flow 4: Bootstrap Token Flow (Headless)" {@@ -196,25 +215,31 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in rawnode_token=$(echo "${token_json}" | jq -r .token)[[ -n "${node_token}" && "${node_token}" != "null" ]]- # 2. Run sam-node join with the bootstrap token+ # 2. Run sam-node join with the bootstrap token; it is a command whose+ # exit code is its verdict.local node_name="${MESH_PREFIX}-node-bootstrap"local data_vol="${MESH_PREFIX}-data-bootstrap"docker volume create "${data_vol}"CLEANUP_VOLUMES+=("${data_vol}")- docker run --name "${node_name}-join" \+ run docker run --name "${node_name}-join" \--network "${MESH_NETWORK}" \$(mesh_get_add_hosts) \-v "${data_vol}:/data" \"sam-node:local" \join --data-dir /data --bootstrap-token "${node_token}" --insecure-control-plane "http://sam-control-plane:8080"+ MESH_CONTAINERS+=("${node_name}-join")+ echo "join output: $output"+ [[ "$status" -eq 0 ]]- # 3. Start the node container with stored identity+ # 3. Start the node container with stored identity. Without an API token+ # the TCP sidecar refuses to start, and the node with it.docker run -d \--name "${node_name}" \--network "${MESH_NETWORK}" \$(mesh_get_add_hosts) \-v "${data_vol}:/data" \+ -e SAM_API_TOKEN="secret-token" \"sam-node:local" \run \--data-dir /data \@@ -222,5 +247,6 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in raw--insecure-control-planeMESH_CONTAINERS+=("${node_name}")- mesh_wait_for_log "${node_name}" "Using stored identity." 20+ mesh_wait_for_mcp_ready bootstrap 30+ assert_enrolled bootstrap BOOTSTRAP}diff --git a/tests/e2e/datapath.bats b/tests/e2e/datapath.batsindex 37312e9f..86e60167 100644--- a/tests/e2e/datapath.bats+++ b/tests/e2e/datapath.bats@@ -76,26 +76,25 @@ EOFecho "[$(date +%T)] Starting Node 1"mesh_start_node 1 "--log-level debug" "${node1_cfg}"local node1_name="${MESH_PREFIX}-node-1"- mesh_wait_for_log "${node1_name}" "SAM Node Online" 20- mesh_wait_for_mcp_ready 1 20-+ mesh_wait_for_mcp_ready 1 30+local node1_peer_id- node1_peer_id=$(docker logs "${node1_name}" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')+ node1_peer_id=$(mesh_node_peer_id 1)+ [[ -n "${node1_peer_id}" ]]# Start Node 2echo "[$(date +%T)] Starting Node 2"mesh_start_node 2 "--log-level debug" "${node2_cfg}"local node2_name="${MESH_PREFIX}-node-2"- mesh_wait_for_log "${node2_name}" "SAM Node Online" 20- mesh_wait_for_mcp_ready 2 20+ mesh_wait_for_mcp_ready 2 30local node2_peer_id- node2_peer_id=$(docker logs "${node2_name}" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')+ node2_peer_id=$(mesh_node_peer_id 2)+ [[ -n "${node2_peer_id}" ]]# Explicitly connect Node 1 to Node 2 (DHT auto-discovery is slow/unreliable in this E2E setup)echo "[$(date +%T)] Explicitly connecting Node 1 to Node 2"- local node2_addr="/dns4/${node2_name}/tcp/5002/p2p/${node2_peer_id}"- run mesh_connect_peer 1 "${node2_addr}"+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"[[ "$status" -eq 0 ]]# Verify connectiondiff --git a/tests/e2e/docker/mock_oidc.py b/tests/e2e/docker/mock_oidc.pyindex 5cb36ca8..12334a4c 100644--- a/tests/e2e/docker/mock_oidc.py+++ b/tests/e2e/docker/mock_oidc.py@@ -91,6 +91,27 @@ class Handler(BaseHTTPRequestHandler):params = urllib.parse.parse_qs(body) if body else {}client_id = params.get('client_id', [''])[0]+ grant_type = params.get('grant_type', [''])[0]++ # Only the grants a SAM component drives are served, so a token+ # is proof of which flow obtained it: client credentials for a+ # configured node, the device grant, with the code handed out+ # above, for an interactive join that has no browser.+ if grant_type == 'urn:ietf:params:oauth:grant-type:device_code':+ granted = params.get('device_code', [''])[0] == 'dev_code_123'+ elif grant_type == 'client_credentials':+ granted = True+ else:+ granted = False+ if not granted:+ error = 'invalid_grant' if grant_type else 'invalid_request'+ data = json.dumps({'error': error}).encode('utf-8')+ self.send_response(400)+ self.send_header('Content-Type', 'application/json')+ self.send_header('Content-Length', str(len(data)))+ self.end_headers()+ self.wfile.write(data)+ return# Assign groups and roles based on client_idgroups = ['data-scientist']diff --git a/tests/e2e/docs_snippets.bats b/tests/e2e/docs_snippets.batsindex 6d32f65c..a8b79f7b 100644--- a/tests/e2e/docs_snippets.bats+++ b/tests/e2e/docs_snippets.bats@@ -24,8 +24,7 @@ teardown() {mesh_start_node 1 "--log-level debug"local node1_name="${MESH_PREFIX}-node-1"- mesh_wait_for_log "${node1_name}" "SAM Node Online" 20- mesh_wait_for_mcp_ready 1 20+ mesh_wait_for_mcp_ready 1 30# Run the agent_demo.py snippet inside a containerrun docker run --rm \diff --git a/tests/e2e/find_remote_tools.bats b/tests/e2e/find_remote_tools.batsindex 17cdae37..2dddd5f4 100644--- a/tests/e2e/find_remote_tools.bats+++ b/tests/e2e/find_remote_tools.bats@@ -20,7 +20,7 @@ start_calc_mcp() {--network-alias calc-mcp \"${CALC_MCP_IMAGE}" >/dev/nullMESH_CONTAINERS+=("${name}")- mesh_wait_for_log "${name}" "Uvicorn running on" 20+ mesh_wait_for_http "http://calc-mcp:7777/mcp" 20}setup() {@@ -40,8 +40,7 @@ teardown() {echo "[$(date +%T)] Starting Node 1"mesh_start_node 1 "--log-level debug"- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60- mesh_wait_for_mcp_ready 1 20+ mesh_wait_for_mcp_ready 1 60echo "[$(date +%T)] Starting calc-mcp backend"start_calc_mcp@@ -50,15 +49,14 @@ teardown() {mesh_start_node 2 \"--log-level debug" \"tests/e2e/docker/calc-mcp/sam-node-config.yaml"- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20- mesh_wait_for_mcp_ready 2 20+ mesh_wait_for_mcp_ready 2 30local node2_peer_id- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')+ node2_peer_id=$(mesh_node_peer_id 2)+ [[ -n "${node2_peer_id}" ]]echo "[$(date +%T)] Connecting Node 1 to Node 2"- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"- run mesh_connect_peer 1 "${node2_addr}"+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"[[ "$status" -eq 0 ]]mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20@@ -92,8 +90,7 @@ teardown() {echo "[$(date +%T)] Starting Node 1"mesh_start_node 1 "--log-level debug"- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60- mesh_wait_for_mcp_ready 1 20+ mesh_wait_for_mcp_ready 1 60echo "[$(date +%T)] Starting calc-mcp backend"start_calc_mcp@@ -102,15 +99,14 @@ teardown() {mesh_start_node 2 \"--log-level debug" \"tests/e2e/docker/calc-mcp/sam-node-config.yaml"- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 60- mesh_wait_for_mcp_ready 2 20+ mesh_wait_for_mcp_ready 2 30local node2_peer_id- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')+ node2_peer_id=$(mesh_node_peer_id 2)+ [[ -n "${node2_peer_id}" ]]echo "[$(date +%T)] Connecting Node 1 to Node 2"- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"- run mesh_connect_peer 1 "${node2_addr}"+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"[[ "$status" -eq 0 ]]mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20@@ -137,8 +133,7 @@ teardown() {echo "[$(date +%T)] Starting Node 1"mesh_start_node 1 "--log-level debug"- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60- mesh_wait_for_mcp_ready 1 20+ mesh_wait_for_mcp_ready 1 60echo "[$(date +%T)] Starting calc-mcp backend"start_calc_mcp@@ -147,15 +142,14 @@ teardown() {mesh_start_node 2 \"--log-level debug" \"tests/e2e/docker/calc-mcp/sam-node-config.yaml"- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20- mesh_wait_for_mcp_ready 2 20+ mesh_wait_for_mcp_ready 2 30local node2_peer_id- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')+ node2_peer_id=$(mesh_node_peer_id 2)+ [[ -n "${node2_peer_id}" ]]echo "[$(date +%T)] Connecting Node 1 to Node 2"- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"- run mesh_connect_peer 1 "${node2_addr}"+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"[[ "$status" -eq 0 ]]mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20@@ -196,8 +190,7 @@ teardown() {echo "[$(date +%T)] Starting Node 1"mesh_start_node 1 "--log-level debug"- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60- mesh_wait_for_mcp_ready 1 20+ mesh_wait_for_mcp_ready 1 60echo "[$(date +%T)] Starting calc-mcp backend"start_calc_mcp@@ -206,15 +199,14 @@ teardown() {mesh_start_node 2 \"--log-level debug" \"tests/e2e/docker/calc-mcp/sam-node-config.yaml"- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20- mesh_wait_for_mcp_ready 2 20+ mesh_wait_for_mcp_ready 2 30local node2_peer_id- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')+ node2_peer_id=$(mesh_node_peer_id 2)+ [[ -n "${node2_peer_id}" ]]echo "[$(date +%T)] Connecting Node 1 to Node 2"- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"- run mesh_connect_peer 1 "${node2_addr}"+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"[[ "$status" -eq 0 ]]mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bashindex e0a21e97..33cd2c9b 100644--- a/tests/e2e/lib/container_mesh.bash+++ b/tests/e2e/lib/container_mesh.bash@@ -77,20 +77,29 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; thenhexdump -vn 32 -e '1/1 "%02x"' /dev/urandom}- mesh_wait_for_log() {- local container="$1"- local needle="$2"- local timeout_s="${3:-20}"+ # mesh_wait_for_http waits until something answers HTTP at url on the test+ # network. The status does not matter, a listener does.+ mesh_wait_for_http() {+ local url="$1"+ local timeout_s="${2:-20}"local i- for ((i=0; i<timeout_s*10; i++)); do- if docker logs "${container}" 2>&1 | grep -Fq "${needle}"; then+ for ((i=0; i<timeout_s; i++)); do+ if docker run --rm --network "${MESH_NETWORK}" "${MESH_RUNTIME_IMAGE}" \+ curl -s -o /dev/null --max-time 5 "${url}"; thenreturn 0fi- sleep 0.1+ sleep 1done+ echo "nothing answered at ${url} within ${timeout_s}s" >&2return 1}+ # mesh_container_log prints a container's output, for failure messages only.+ mesh_container_log() {+ echo "--- ${1} ---" >&2+ docker logs --tail "${2:-80}" "$1" >&2 2>&1 || true+ }+# Pods matching a selector that will never become ready on their own.# Deliberately excludes CrashLoopBackOff: the control plane restarts a few# times while the database comes up, so treating that as terminal would swap@@ -183,19 +192,63 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; thendone}+ # mesh_wait_for_mcp_ready returns once node <idx> answers on its+ # authenticated API, which the node serves only after it enrolled and a+ # router admitted it. A node that exited first fails at once.mesh_wait_for_mcp_ready() {local idx="$1"local timeout_s="${2:-20}"+ local name="${MESH_PREFIX}-node-${idx}"local ifor ((i=0; i<timeout_s; i++)); do- if docker run --rm --network "${MESH_NETWORK}" python:3.12 curl -s -X POST -H "Content-Type: application/json" -H "X-Sam-Authentication: Bearer secret-token" -d '{"jsonrpc":"2.0","method":"ping","id":1}' --max-time 5 -D - http://${MESH_PREFIX}-node-${idx}:8080/mcp | grep -q "200 OK"; then+ if [[ "$(docker inspect -f '{{.State.Running}}' "${name}" 2>/dev/null)" != "true" ]]; then+ echo "${name} exited before serving its API" >&2+ mesh_container_log "${name}"+ return 1+ fi+ if docker run --rm --network "${MESH_NETWORK}" python:3.12 curl -s -X POST -H "Content-Type: application/json" -H "X-Sam-Authentication: Bearer secret-token" -d '{"jsonrpc":"2.0","method":"ping","id":1}' --max-time 5 -D - http://${name}:8080/mcp | grep -q "200 OK"; thenreturn 0fisleep 1done+ echo "${name} did not serve its API within ${timeout_s}s" >&2+ mesh_container_log "${name}"return 1}+ # mesh_node_peer_id is node <idx>'s peer ID, as the node reports it.+ mesh_node_peer_id() {+ local idx="$1"+ timeout 15s docker run --rm --network "${MESH_NETWORK}" "${MESH_RUNTIME_IMAGE}" \+ mcp-client -url "http://${MESH_PREFIX}-node-${idx}:8080/mcp" -tool "get_mesh_info" 2>/dev/null |+ jq -r '.peer_id // empty'+ }++ # mesh_node_addr is the address a peer dials node <idx> on within the test+ # network.+ mesh_node_addr() {+ local idx="$1"+ local peer_id+ peer_id="$(mesh_node_peer_id "${idx}")"+ [[ -n "${peer_id}" ]] || return 1+ echo "/dns4/${MESH_PREFIX}-node-${idx}/tcp/5002/p2p/${peer_id}"+ }++ # mesh_admin_status is the control plane's view of the mesh, as it+ # serialises it on /admin/status.+ mesh_admin_status() {+ docker run --rm --network "${MESH_NETWORK}" $(mesh_get_add_hosts) "${MESH_RUNTIME_IMAGE}" \+ curl -sf --max-time 10 -H "Authorization: Bearer super-secret-admin-token" \+ "http://sam-control-plane:8080/admin/status"+ }++ # mesh_enrolled_node prints the control plane's enrollment record for a+ # peer, or nothing.+ mesh_enrolled_node() {+ local peer_id="$1"+ mesh_admin_status | jq -c --arg id "${peer_id}" '.enrolled_nodes[] | select(.PeerID == $id)'+ }+# POST /debug/connect-peer on node <idx>; the REST endpoint that replaced# the connect_peer MCP tool (#318). Nonzero exit on any non-2xx response.mesh_connect_peer() {@@ -420,31 +473,26 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; thenfimesh_wait_for_rollout statefulset/sam-router- local i- for ((i=0; i<200; i++)); do- if kubectl --context="${KUBECONTEXT}" logs "sam-router-0" 2>&1 | grep -q "PeerID:"; then- break- fi- sleep 0.1- done- local router_peer_id- router_peer_id=$(kubectl --context="${KUBECONTEXT}" logs "sam-router-0" | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1 || true)- [[ -n "${router_peer_id}" ]]-- # The router pod reports Ready before its lease reaches the control plane, and- # a node's /register serves router addresses from that lease, so a node- # started in between enrolls against an empty list and exits.+ # The router pod reports Ready before its lease reaches the control plane,+ # and a node's /register serves router addresses from that lease, so a+ # node started in between enrolls against an empty list and exits. The+ # lease is also where the router's peer ID comes from; on a reused+ # cluster the freshest renewal is the router that is running now.local router_node_iprouter_node_ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"${MESH_NETWORK:-kind}\").IPAddress}}" \"$(kubectl --context="${KUBECONTEXT}" get pod sam-router-0 -o jsonpath='{.spec.nodeName}')")+ local router_peer_id=""local lease_deadline=$((SECONDS + 60))- until docker run --rm --network "${MESH_NETWORK:-kind}" python:3.12 \- curl -sf --max-time 5 "http://${router_node_ip}:8080/info" 2>/dev/null | grep -qaF "${router_peer_id}"; do- if ((SECONDS >= lease_deadline)); then+ while [[ -z "${router_peer_id}" ]]; do+ router_peer_id=$(docker run --rm --network "${MESH_NETWORK:-kind}" "${MESH_RUNTIME_IMAGE}" \+ curl -sf --max-time 5 -H "Authorization: Bearer super-secret-admin-token" \+ "http://${router_node_ip}:8080/admin/status" 2>/dev/null |+ jq -r '.active_routers | sort_by(.LastRenewal) | last | .PeerID // empty')+ if [[ -z "${router_peer_id}" ]] && ((SECONDS >= lease_deadline)); thenecho "router lease did not reach the control plane within 60s" >&2return 1fi- sleep 1+ [[ -n "${router_peer_id}" ]] || sleep 1doneecho "${router_peer_id}" > "/tmp/sam-wi-test-router-peer-id"diff --git a/tests/e2e/policy.bats b/tests/e2e/policy.batsindex a9878964..c6572949 100644--- a/tests/e2e/policy.bats+++ b/tests/e2e/policy.bats@@ -20,122 +20,9 @@ start_calc_mcp() {--network-alias calc-mcp \"${CALC_MCP_IMAGE}" >/dev/nullMESH_CONTAINERS+=("${name}")- mesh_wait_for_log "${name}" "Uvicorn running on" 20+ mesh_wait_for_http "http://calc-mcp:7777/mcp" 20}-# Custom mock OIDC server that returns 'data-scientist' role-mesh_start_mock_oidc_custom() {- local name="${MESH_PREFIX}-oidc"- local cmd- read -r -d '' cmd <<'EOF' || true-python3 - <<'PY'-import json-import time-import jwt-from http.server import BaseHTTPRequestHandler, HTTPServer--# The signing key is generated fresh on every start and the JWKS derived from-# it, so no private key lives in the repository: an issuer whose key is public-# would let anyone mint a token for any identity a control plane trusts.-from cryptography.hazmat.primitives import serialization-from cryptography.hazmat.primitives.asymmetric import rsa--def _b64url_uint(n):- import base64- raw = n.to_bytes((n.bit_length() + 7) // 8, 'big')- return base64.urlsafe_b64encode(raw).rstrip(b'=').decode('ascii')--_KEY = rsa.generate_private_key(public_exponent=65537, key_size=2048)-PRIVATE_KEY = _KEY.private_bytes(- serialization.Encoding.PEM,- serialization.PrivateFormat.PKCS8,- serialization.NoEncryption(),-)-_PUB = _KEY.public_key().public_numbers()-JWKS = {- "keys": [- {- "kty": "RSA",- "alg": "RS256",- "use": "sig",- "kid": "test-key-id",- "n": _b64url_uint(_PUB.n),- "e": _b64url_uint(_PUB.e),- }- ]-}--class Handler(BaseHTTPRequestHandler):- def do_GET(self):- if self.path == '/.well-known/openid-configuration':- body = {- 'issuer': 'http://mock-oidc:18080',- 'authorization_endpoint': 'http://mock-oidc:18080/auth',- 'token_endpoint': 'http://mock-oidc:18080/token',- 'jwks_uri': 'http://mock-oidc:18080/keys'- }- data = json.dumps(body).encode('utf-8')- self.send_response(200)- self.send_header('Content-Type', 'application/json')- self.send_header('Content-Length', str(len(data)))- self.end_headers()- self.wfile.write(data)- return- if self.path == '/keys':- data = json.dumps(JWKS).encode('utf-8')- self.send_response(200)- self.send_header('Content-Type', 'application/json')- self.send_header('Content-Length', str(len(data)))- self.end_headers()- self.wfile.write(data)- return- self.send_response(200)- self.end_headers()- self.wfile.write(b'ok')-- def do_POST(self):- if self.path == '/token':- payload = {- 'iss': 'http://mock-oidc:18080',- 'aud': 'sam-e2e',- 'sub': 'test-user',- 'exp': int(time.time()) + 3600,- 'roles': ['data-scientist'] # Custom role- }- token = jwt.encode(payload, PRIVATE_KEY, algorithm='RS256', headers={'kid': 'test-key-id'})- body = {- 'access_token': token,- 'token_type': 'Bearer',- 'expires_in': 3600- }- data = json.dumps(body).encode('utf-8')- self.send_response(200)- self.send_header('Content-Type', 'application/json')- self.send_header('Content-Length', str(len(data)))- self.end_headers()- self.wfile.write(data)- return- self.send_response(404)- self.end_headers()--print("Mock OIDC server ready", flush=True)-HTTPServer(('0.0.0.0', 18080), Handler).serve_forever()-PY-EOF-- docker run -d \- --name "${name}" \- --network "${MESH_NETWORK}" \- --network-alias mock-oidc \- sam-mock-oidc:local \- sh -c "${cmd}" >/dev/null-- MESH_CONTAINERS+=("${name}")- mesh_wait_for_log "${name}" "Mock OIDC server ready" 30-}---setup() {mesh_setup_envbuild_calc_mcp_image@@ -166,43 +53,20 @@ attenuation:# Start Node 1 (Target) with local policy filemesh_start_node 1 "" "${config_file}"- mesh_wait_for_log "${MESH_PREFIX}-node-1" "Successfully enrolled" 20+ mesh_wait_for_mcp_ready 1 30++ export TARGET_PEER_ID+ TARGET_PEER_ID="$(mesh_node_peer_id 1)"+ [[ -n "${TARGET_PEER_ID}" ]]# Start Node 2 (Caller)mesh_start_node 2- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20- mesh_wait_for_mcp_ready 2-- local node2_id- node2_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)-- # Wait for discovery (Node 2 should see Node 1)- local i- local router_id- router_id="$(cat "/tmp/${MESH_PREFIX}-router-peer-id")"- export TARGET_PEER_ID=""-- for ((i=0; i<40; i++)); do- local output- output="$(docker run --rm --network "${MESH_NETWORK}" "${MESH_RUNTIME_IMAGE}" mcp-client -url "http://${MESH_PREFIX}-node-2:8080/mcp" -tool "get_mesh_info")"- TARGET_PEER_ID=$(echo "${output}" | grep -oE '12D3Koo[a-zA-Z0-9]+' | grep -v "${router_id}" | grep -v "${node2_id}" | head -n 1)- if [[ -n "${TARGET_PEER_ID}" ]]; then- break- fi- sleep 1- done-- echo "Node 2 logs after discovery loop:" >&3- docker logs "${MESH_PREFIX}-node-2" >&3-- if [[ -z "${TARGET_PEER_ID}" ]]; then- echo "Timeout waiting for discovery of Node 1"- return 1- fi+ mesh_wait_for_mcp_ready 2 30- # Explicitly connect Node 2 to Node 1 to avoid "no addresses" error- local node1_addr="/dns4/${MESH_PREFIX}-node-1/tcp/5002/p2p/${TARGET_PEER_ID}"- mesh_connect_peer 2 "${node1_addr}" >/dev/null+ # Connect Node 2 to Node 1 explicitly so the calls below do not depend on+ # discovery timing, and wait until Node 2 reports the connection.+ mesh_connect_peer 2 "$(mesh_node_addr 1)" >/dev/null+ mesh_wait_for_peer_connection 2 "${TARGET_PEER_ID}" 30}teardown() {diff --git a/tests/e2e/python_sdk_test.bats b/tests/e2e/python_sdk_test.batsindex 36e7e22b..79cb7c5c 100644--- a/tests/e2e/python_sdk_test.bats+++ b/tests/e2e/python_sdk_test.bats@@ -25,8 +25,7 @@ teardown() {[[ "$status" -eq 0 ]]local node1_name="${MESH_PREFIX}-node-1"- mesh_wait_for_log "${node1_name}" "SAM Node Online" 20- mesh_wait_for_mcp_ready 1 20+ mesh_wait_for_mcp_ready 1 30# Use the Python SDK to interact with the noderun docker run --rm \diff --git a/tests/e2e/relay.bats b/tests/e2e/relay.batsindex bed05813..8496ba7b 100644--- a/tests/e2e/relay.bats+++ b/tests/e2e/relay.bats@@ -36,21 +36,6 @@ teardown() {CLEANUP_NETWORKS=()}-@test "node starts with --enable-relay=true and logs message" {- run mesh_start_mock_oidc- [[ "$status" -eq 0 ]]-- run mesh_start_router- [[ "$status" -eq 0 ]]-- # Start node with relay flag- mesh_start_node "1" "--enable-relay=true --log-level=debug"-- # Wait for log message- run mesh_wait_for_log "${MESH_PREFIX}-node-1" "Enabling Relay Service" 20- [[ "$status" -eq 0 ]]-}-@test "nodes in isolated networks discover and connect via relay" {run mesh_start_mock_oidc[[ "$status" -eq 0 ]]@@ -123,23 +108,20 @@ EOF# Gate on MCP actually answering: get_mesh_info polls below go through it, and# a not-yet-listening sidecar burns discovery budget on empty responses.- run mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 30- [[ "$status" -eq 0 ]]run mesh_wait_for_mcp_ready "1" 30[[ "$status" -eq 0 ]]+ local node1_peer_id node2_peer_id+ node1_peer_id=$(mesh_node_peer_id 1)+ [[ -n "${node1_peer_id}" ]]OLD_NET=$MESH_NETWORKMESH_NETWORK=$MESH_NETWORK_2- run mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 30- [[ "$status" -eq 0 ]]run mesh_wait_for_mcp_ready "2" 30[[ "$status" -eq 0 ]]+ node2_peer_id=$(mesh_node_peer_id 2)+ [[ -n "${node2_peer_id}" ]]MESH_NETWORK=$OLD_NET- local node1_peer_id node2_peer_id- node1_peer_id=$(docker logs "${MESH_PREFIX}-node-1" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')-# Assert on the specific peer, not on set size: a relayed connection can leave# extra entries in connected_peers, so "length - 1" is not a stable count.run mesh_wait_for_peer_connection "1" "${node2_peer_id}" 60diff --git a/tests/e2e/services.bats b/tests/e2e/services.batsindex 58b44977..e47bac04 100644--- a/tests/e2e/services.bats+++ b/tests/e2e/services.bats@@ -20,7 +20,7 @@ start_calc_mcp() {--network-alias calc-mcp \"${CALC_MCP_IMAGE}" >/dev/nullMESH_CONTAINERS+=("${name}")- mesh_wait_for_log "${name}" "Uvicorn running on" 20+ mesh_wait_for_http "http://calc-mcp:7777/mcp" 20}setup() {@@ -41,8 +41,7 @@ teardown() {echo "[$(date +%T)] Starting Node 1"mesh_start_node 1 "--log-level debug"local node1_name="${MESH_PREFIX}-node-1"- mesh_wait_for_log "${node1_name}" "SAM Node Online" 60- mesh_wait_for_mcp_ready 1 20+ mesh_wait_for_mcp_ready 1 60echo "[$(date +%T)] Starting calc-mcp backend"start_calc_mcp@@ -52,15 +51,14 @@ teardown() {"--log-level debug" \"tests/e2e/docker/calc-mcp/sam-node-config.yaml"local node2_name="${MESH_PREFIX}-node-2"- mesh_wait_for_log "${node2_name}" "SAM Node Online" 20- mesh_wait_for_mcp_ready 2 20+ mesh_wait_for_mcp_ready 2 30local node2_peer_id- node2_peer_id=$(docker logs "${node2_name}" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')+ node2_peer_id=$(mesh_node_peer_id 2)+ [[ -n "${node2_peer_id}" ]]echo "[$(date +%T)] Connecting Node 1 to Node 2"- local node2_addr="/dns4/${node2_name}/tcp/5002/p2p/${node2_peer_id}"- run mesh_connect_peer 1 "${node2_addr}"+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"[[ "$status" -eq 0 ]]mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20diff --git a/tests/e2e/standalone.bats b/tests/e2e/standalone.batsindex 22440142..dc3c7a27 100644--- a/tests/e2e/standalone.bats+++ b/tests/e2e/standalone.bats@@ -51,6 +51,33 @@ wait_for_log() {return 1}+# wait_for_node waits until the node named $1 serves its API on its socket,+# which it does only after it enrolled, a router admitted it, and every+# service in its configuration was registered. A node that exited fails at+# once, with its log.+wait_for_node() {+ local name="$1" pid_var="NODE_${1^^}_PID"+ local sock="$TEST_TMPDIR/node-$name/sam.sock"+ for _ in $(seq 1 150); do+ if ! kill -0 "${!pid_var}" 2>/dev/null; then+ echo "node $name exited before serving its API:" >&2+ cat "$TEST_TMPDIR/node-$name.log" >&2 || true+ return 1+ fi+ curl -sf --unix-socket "$sock" http://localhost/healthz > /dev/null 2>&1 && return 0+ sleep 0.2+ done+ echo "node $name did not serve its API in time:" >&2+ cat "$TEST_TMPDIR/node-$name.log" >&2 || true+ return 1+}++# node_peer_id is the peer ID node $1 reports for itself.+node_peer_id() {+ curl -sf --unix-socket "$TEST_TMPDIR/node-$1/sam.sock" http://localhost/debug/mesh-info |+ python3 -c 'import json, sys; print(json.load(sys.stdin)["peer_id"])'+}+# start_node boots a background sam-node joined via the bootstrap token; sets# NODE_<NAME>_PID for teardown. The sidecar serves only on the Unix socket so# two nodes on one host never fight over the default TCP bind, and loopback@@ -117,18 +144,15 @@ EOFstart_node a --config "$TEST_TMPDIR/node-a-services.yaml"start_node b- wait_for_log "$TEST_TMPDIR/node-a.log" "SAM Node Online"- wait_for_log "$TEST_TMPDIR/node-b.log" "SAM Node Online"- peer_a="$(grep -oE 'PeerID: [A-Za-z0-9]+' "$TEST_TMPDIR/node-a.log" | head -1 | cut -d' ' -f2)"+ wait_for_node a+ wait_for_node b+ peer_a="$(node_peer_id a)"[[ -n "$peer_a" ]]sock_a="$TEST_TMPDIR/node-a/sam.sock"sock_b="$TEST_TMPDIR/node-b/sam.sock"[[ -S "$sock_a" && -S "$sock_b" ]]- # The mislabelled service is registered but withheld from discovery.- wait_for_log "$TEST_TMPDIR/node-a.log" "not-an-mcp-server but not advertising it"-# Node B's OpenAI-compatible endpoint lists the model with A as its owner:# the announcement crosses A -> router -> B over the mesh.models=""@@ -148,7 +172,9 @@ EOF[[ "$status" -eq 0 ]][[ "$output" == *"hello from the mesh"* ]]- # The plain HTTP server never shows up as an MCP provider.+ # The plain HTTP server never shows up as an MCP provider: node A had+ # registered every configured service before its API came up, so its+ # absence here is a verdict, not a race.run curl -sf --unix-socket "$sock_b" \"http://localhost/sam/service/discover?type=mcp&name=not-an-mcp-server&timeout=3s"[[ "$status" -eq 0 ]]