Skip to content

Agent Mesh

e2e: watch the mesh, not the containers' logs

Antonio Ojeac63b27f04cc3
diff --git a/tests/e2e/a2a_mesh.bats b/tests/e2e/a2a_mesh.bats
index 402535cb..313e9aad 100644
--- a/tests/e2e/a2a_mesh.bats
+++ b/tests/e2e/a2a_mesh.bats
@@ -20,7 +20,7 @@ start_a2a_echo() {
--network-alias a2a-echo \
"${A2A_ECHO_IMAGE}" >/dev/null
MESH_CONTAINERS+=("${name}")
- mesh_wait_for_log "${name}" "Uvicorn running on" 30
+ mesh_wait_for_http "http://a2a-echo:7777/" 30
}
setup() {
@@ -43,8 +43,7 @@ teardown() {
echo "[$(date +%T)] Starting Node 1 (consumer)"
mesh_start_node 1 "--log-level debug"
- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60
- mesh_wait_for_mcp_ready 1 20
+ mesh_wait_for_mcp_ready 1 60
echo "[$(date +%T)] Starting a2a echo agent backend"
start_a2a_echo
@@ -53,15 +52,14 @@ teardown() {
mesh_start_node 2 \
"--log-level debug" \
"tests/e2e/docker/a2a-echo/sam-node-config.yaml"
- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 2 20
+ mesh_wait_for_mcp_ready 2 30
local node2_peer_id
- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
+ node2_peer_id=$(mesh_node_peer_id 2)
+ [[ -n "${node2_peer_id}" ]]
echo "[$(date +%T)] Connecting Node 1 to Node 2"
- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"
- run mesh_connect_peer 1 "${node2_addr}"
+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"
[[ "$status" -eq 0 ]]
mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20
diff --git a/tests/e2e/agent_sandbox.bats b/tests/e2e/agent_sandbox.bats
index 1d394701..12ad0c9d 100644
--- a/tests/e2e/agent_sandbox.bats
+++ b/tests/e2e/agent_sandbox.bats
@@ -37,8 +37,7 @@ agent_curl() {
# HOME is unwritable and the default location cannot be created.
mesh_start_node 1 "--socket-path /sockets/node.sock --data-dir /sockets/node-data" "" \
"-v ${MESH_SOCKET_DIR}:/sockets --user $(id -u):$(id -g) -e HOME=/sockets"
- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60
- mesh_wait_for_mcp_ready 1 20
+ mesh_wait_for_mcp_ready 1 60
# The gateway consumes the node. It has no network either: the node's socket
# is the only thing it needs.
diff --git a/tests/e2e/auth_flows.bats b/tests/e2e/auth_flows.bats
index 145c37dd..80e58674 100644
--- a/tests/e2e/auth_flows.bats
+++ b/tests/e2e/auth_flows.bats
@@ -10,6 +10,24 @@ teardown() {
mesh_cleanup_env
}
+# assert_enrolled fails unless the control plane holds an enrollment for the
+# node named <idx>, made the given way; it prints the record.
+assert_enrolled() {
+ local idx="$1"
+ local enrollment_type="$2"
+ local peer_id record
+ peer_id="$(mesh_node_peer_id "${idx}")"
+ [[ -n "${peer_id}" ]]
+ record="$(mesh_enrolled_node "${peer_id}")"
+ if [[ -z "${record}" ]]; then
+ echo "control plane has no enrollment for ${peer_id}" >&2
+ return 1
+ fi
+ [[ "$(jq -r '.EnrollmentType' <<<"${record}")" == "${enrollment_type}" ]]
+ [[ "$(jq -r '.Role' <<<"${record}")" == "sam:role:node" ]]
+ echo "${record}"
+}
+
@test "Authentication Flow 1: Client Credentials Flow" {
run mesh_start_mock_oidc
[[ "$status" -eq 0 ]]
@@ -20,8 +38,8 @@ teardown() {
# mesh_start_node uses --token-url by default, which implements Client Credentials flow
mesh_start_node 1
- run mesh_assert_container_running "${MESH_PREFIX}-node-1"
- [[ "$status" -eq 0 ]]
+ mesh_wait_for_mcp_ready 1 30
+ assert_enrolled 1 OIDC
}
@test "Authentication Flow 2: Device Authorization Flow (Interactive)" {
@@ -31,16 +49,12 @@ teardown() {
run mesh_start_router
[[ "$status" -eq 0 ]]
- # 1. Get a token from mock provider using Python helper
- local token
- token=$(docker run --rm --network "${MESH_NETWORK}" $(mesh_get_add_hosts) python:3.12 python3 -c "import urllib.request; import json; req = urllib.request.Request('http://mock-oidc:18080/token', data=b''); resp = urllib.request.urlopen(req); print(json.loads(resp.read().decode())['access_token'])")
-
- [[ -n "${token}" ]]
-
- # 2. Run sam-node join to enroll and store identity. The sam-node image has
+ # Run sam-node join to enroll and store identity. The sam-node image has
# no browser to open, so the loopback flow can't complete; join falls back
# to the OAuth 2.0 Device Authorization Grant (RFC 8628), which the mock
- # OIDC provider advertises and auto-approves on the first poll.
+ # OIDC provider advertises and auto-approves on the first poll. The
+ # provider serves the device grant only for the code it handed out, so an
+ # enrollment is proof the device flow ran.
local node_name="${MESH_PREFIX}-node-login"
local router_peer_id
router_peer_id=$(cat "/tmp/${MESH_PREFIX}-router-peer-id")
@@ -63,15 +77,11 @@ teardown() {
join --config /etc/sam/node-config.yaml --data-dir /data --insecure-control-plane "http://sam-control-plane:8080"
MESH_CONTAINERS+=("${node_name}-join")
- run mesh_wait_for_log "${node_name}-join" "OAuth Device Authorization Flow" 20
- [[ "$status" -eq 0 ]]
- run mesh_wait_for_log "${node_name}-join" "Enter code: ABCD-1234" 20
- [[ "$status" -eq 0 ]]
-
- # Wait for the join process to finish and check it succeeded.
- run mesh_wait_for_log "${node_name}-join" "Successfully joined the Sovereign Agent Mesh!" 20
- [[ "$status" -eq 0 ]]
- [[ "$(docker inspect -f '{{.State.ExitCode}}' "${node_name}-join")" -eq 0 ]]
+ # join is a command: it ends, and its exit code is its verdict.
+ run timeout 60 docker wait "${node_name}-join"
+ echo "join exit: $output"
+ [[ "$status" -eq 0 && "$output" == "0" ]] || mesh_container_log "${node_name}-join"
+ [[ "$status" -eq 0 && "$output" == "0" ]]
docker rm -f "${node_name}-join" >/dev/null 2>&1 || true
# Now run the node with the stored identity, on the same control plane it
@@ -89,8 +99,13 @@ teardown() {
--insecure-control-plane
MESH_CONTAINERS+=("${node_name}")
- mesh_wait_for_log "${node_name}" "Using stored identity." 20
- mesh_wait_for_log "${node_name}" "Serving the local API on Unix socket" 30
+ mesh_wait_for_mcp_ready login 30
+
+ # Enrolled by the user the provider signed in, through the device grant.
+ local record
+ record="$(assert_enrolled login OIDC)"
+ echo "enrollment: ${record}"
+ [[ "$(jq -r '.ClaimsJSON | fromjson | .sub' <<<"${record}")" == "test-user" ]]
# The labels join declared must come back attested. /sam/identity hands back
# the raw biscuit rather than decoded claims, so read the signed
@@ -123,9 +138,12 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in raw
run mesh_start_router
[[ "$status" -eq 0 ]]
- # 1. Get a token from mock provider
+ # 1. Get a token from the mock provider the way a workload platform would
+ # mint one: a real grant, not an empty POST.
local token
- token=$(docker run --rm --network "${MESH_NETWORK}" $(mesh_get_add_hosts) python:3.12 python3 -c "import urllib.request; import json; req = urllib.request.Request('http://mock-oidc:18080/token', data=b''); resp = urllib.request.urlopen(req); print(json.loads(resp.read().decode())['access_token'])")
+ token=$(docker run --rm --network "${MESH_NETWORK}" $(mesh_get_add_hosts) "${MESH_RUNTIME_IMAGE}" \
+ curl -sf --max-time 10 -X POST -d 'grant_type=client_credentials&client_id=sam-mesh-audience' \
+ http://mock-oidc:18080/token | jq -r '.access_token // empty')
[[ -n "${token}" ]]
@@ -157,7 +175,8 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in raw
--jwt-path "/var/run/secrets/tokens/sa-token"
MESH_CONTAINERS+=("${node_name}")
- mesh_wait_for_log "${node_name}" "SAM Node Online" 20
+ mesh_wait_for_mcp_ready wi 30
+ assert_enrolled wi OIDC
}
@test "Authentication Flow 4: Bootstrap Token Flow (Headless)" {
@@ -196,25 +215,31 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in raw
node_token=$(echo "${token_json}" | jq -r .token)
[[ -n "${node_token}" && "${node_token}" != "null" ]]
- # 2. Run sam-node join with the bootstrap token
+ # 2. Run sam-node join with the bootstrap token; it is a command whose
+ # exit code is its verdict.
local node_name="${MESH_PREFIX}-node-bootstrap"
local data_vol="${MESH_PREFIX}-data-bootstrap"
docker volume create "${data_vol}"
CLEANUP_VOLUMES+=("${data_vol}")
- docker run --name "${node_name}-join" \
+ run docker run --name "${node_name}-join" \
--network "${MESH_NETWORK}" \
$(mesh_get_add_hosts) \
-v "${data_vol}:/data" \
"sam-node:local" \
join --data-dir /data --bootstrap-token "${node_token}" --insecure-control-plane "http://sam-control-plane:8080"
+ MESH_CONTAINERS+=("${node_name}-join")
+ echo "join output: $output"
+ [[ "$status" -eq 0 ]]
- # 3. Start the node container with stored identity
+ # 3. Start the node container with stored identity. Without an API token
+ # the TCP sidecar refuses to start, and the node with it.
docker run -d \
--name "${node_name}" \
--network "${MESH_NETWORK}" \
$(mesh_get_add_hosts) \
-v "${data_vol}:/data" \
+ -e SAM_API_TOKEN="secret-token" \
"sam-node:local" \
run \
--data-dir /data \
@@ -222,5 +247,6 @@ sys.exit(0 if b'\x05label' in raw and b'\x06region' in raw and b'\x02eu' in raw
--insecure-control-plane
MESH_CONTAINERS+=("${node_name}")
- mesh_wait_for_log "${node_name}" "Using stored identity." 20
+ mesh_wait_for_mcp_ready bootstrap 30
+ assert_enrolled bootstrap BOOTSTRAP
}
diff --git a/tests/e2e/datapath.bats b/tests/e2e/datapath.bats
index 37312e9f..86e60167 100644
--- a/tests/e2e/datapath.bats
+++ b/tests/e2e/datapath.bats
@@ -76,26 +76,25 @@ EOF
echo "[$(date +%T)] Starting Node 1"
mesh_start_node 1 "--log-level debug" "${node1_cfg}"
local node1_name="${MESH_PREFIX}-node-1"
- mesh_wait_for_log "${node1_name}" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 1 20
-
+ mesh_wait_for_mcp_ready 1 30
+
local node1_peer_id
- node1_peer_id=$(docker logs "${node1_name}" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
+ node1_peer_id=$(mesh_node_peer_id 1)
+ [[ -n "${node1_peer_id}" ]]
# Start Node 2
echo "[$(date +%T)] Starting Node 2"
mesh_start_node 2 "--log-level debug" "${node2_cfg}"
local node2_name="${MESH_PREFIX}-node-2"
- mesh_wait_for_log "${node2_name}" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 2 20
+ mesh_wait_for_mcp_ready 2 30
local node2_peer_id
- node2_peer_id=$(docker logs "${node2_name}" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
+ node2_peer_id=$(mesh_node_peer_id 2)
+ [[ -n "${node2_peer_id}" ]]
# Explicitly connect Node 1 to Node 2 (DHT auto-discovery is slow/unreliable in this E2E setup)
echo "[$(date +%T)] Explicitly connecting Node 1 to Node 2"
- local node2_addr="/dns4/${node2_name}/tcp/5002/p2p/${node2_peer_id}"
- run mesh_connect_peer 1 "${node2_addr}"
+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"
[[ "$status" -eq 0 ]]
# Verify connection
diff --git a/tests/e2e/docker/mock_oidc.py b/tests/e2e/docker/mock_oidc.py
index 5cb36ca8..12334a4c 100644
--- a/tests/e2e/docker/mock_oidc.py
+++ b/tests/e2e/docker/mock_oidc.py
@@ -91,6 +91,27 @@ class Handler(BaseHTTPRequestHandler):
params = urllib.parse.parse_qs(body) if body else {}
client_id = params.get('client_id', [''])[0]
+ grant_type = params.get('grant_type', [''])[0]
+
+ # Only the grants a SAM component drives are served, so a token
+ # is proof of which flow obtained it: client credentials for a
+ # configured node, the device grant, with the code handed out
+ # above, for an interactive join that has no browser.
+ if grant_type == 'urn:ietf:params:oauth:grant-type:device_code':
+ granted = params.get('device_code', [''])[0] == 'dev_code_123'
+ elif grant_type == 'client_credentials':
+ granted = True
+ else:
+ granted = False
+ if not granted:
+ error = 'invalid_grant' if grant_type else 'invalid_request'
+ data = json.dumps({'error': error}).encode('utf-8')
+ self.send_response(400)
+ self.send_header('Content-Type', 'application/json')
+ self.send_header('Content-Length', str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+ return
# Assign groups and roles based on client_id
groups = ['data-scientist']
diff --git a/tests/e2e/docs_snippets.bats b/tests/e2e/docs_snippets.bats
index 6d32f65c..a8b79f7b 100644
--- a/tests/e2e/docs_snippets.bats
+++ b/tests/e2e/docs_snippets.bats
@@ -24,8 +24,7 @@ teardown() {
mesh_start_node 1 "--log-level debug"
local node1_name="${MESH_PREFIX}-node-1"
- mesh_wait_for_log "${node1_name}" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 1 20
+ mesh_wait_for_mcp_ready 1 30
# Run the agent_demo.py snippet inside a container
run docker run --rm \
diff --git a/tests/e2e/find_remote_tools.bats b/tests/e2e/find_remote_tools.bats
index 17cdae37..2dddd5f4 100644
--- a/tests/e2e/find_remote_tools.bats
+++ b/tests/e2e/find_remote_tools.bats
@@ -20,7 +20,7 @@ start_calc_mcp() {
--network-alias calc-mcp \
"${CALC_MCP_IMAGE}" >/dev/null
MESH_CONTAINERS+=("${name}")
- mesh_wait_for_log "${name}" "Uvicorn running on" 20
+ mesh_wait_for_http "http://calc-mcp:7777/mcp" 20
}
setup() {
@@ -40,8 +40,7 @@ teardown() {
echo "[$(date +%T)] Starting Node 1"
mesh_start_node 1 "--log-level debug"
- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60
- mesh_wait_for_mcp_ready 1 20
+ mesh_wait_for_mcp_ready 1 60
echo "[$(date +%T)] Starting calc-mcp backend"
start_calc_mcp
@@ -50,15 +49,14 @@ teardown() {
mesh_start_node 2 \
"--log-level debug" \
"tests/e2e/docker/calc-mcp/sam-node-config.yaml"
- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 2 20
+ mesh_wait_for_mcp_ready 2 30
local node2_peer_id
- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
+ node2_peer_id=$(mesh_node_peer_id 2)
+ [[ -n "${node2_peer_id}" ]]
echo "[$(date +%T)] Connecting Node 1 to Node 2"
- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"
- run mesh_connect_peer 1 "${node2_addr}"
+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"
[[ "$status" -eq 0 ]]
mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20
@@ -92,8 +90,7 @@ teardown() {
echo "[$(date +%T)] Starting Node 1"
mesh_start_node 1 "--log-level debug"
- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60
- mesh_wait_for_mcp_ready 1 20
+ mesh_wait_for_mcp_ready 1 60
echo "[$(date +%T)] Starting calc-mcp backend"
start_calc_mcp
@@ -102,15 +99,14 @@ teardown() {
mesh_start_node 2 \
"--log-level debug" \
"tests/e2e/docker/calc-mcp/sam-node-config.yaml"
- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 60
- mesh_wait_for_mcp_ready 2 20
+ mesh_wait_for_mcp_ready 2 30
local node2_peer_id
- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
+ node2_peer_id=$(mesh_node_peer_id 2)
+ [[ -n "${node2_peer_id}" ]]
echo "[$(date +%T)] Connecting Node 1 to Node 2"
- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"
- run mesh_connect_peer 1 "${node2_addr}"
+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"
[[ "$status" -eq 0 ]]
mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20
@@ -137,8 +133,7 @@ teardown() {
echo "[$(date +%T)] Starting Node 1"
mesh_start_node 1 "--log-level debug"
- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60
- mesh_wait_for_mcp_ready 1 20
+ mesh_wait_for_mcp_ready 1 60
echo "[$(date +%T)] Starting calc-mcp backend"
start_calc_mcp
@@ -147,15 +142,14 @@ teardown() {
mesh_start_node 2 \
"--log-level debug" \
"tests/e2e/docker/calc-mcp/sam-node-config.yaml"
- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 2 20
+ mesh_wait_for_mcp_ready 2 30
local node2_peer_id
- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
+ node2_peer_id=$(mesh_node_peer_id 2)
+ [[ -n "${node2_peer_id}" ]]
echo "[$(date +%T)] Connecting Node 1 to Node 2"
- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"
- run mesh_connect_peer 1 "${node2_addr}"
+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"
[[ "$status" -eq 0 ]]
mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20
@@ -196,8 +190,7 @@ teardown() {
echo "[$(date +%T)] Starting Node 1"
mesh_start_node 1 "--log-level debug"
- mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 60
- mesh_wait_for_mcp_ready 1 20
+ mesh_wait_for_mcp_ready 1 60
echo "[$(date +%T)] Starting calc-mcp backend"
start_calc_mcp
@@ -206,15 +199,14 @@ teardown() {
mesh_start_node 2 \
"--log-level debug" \
"tests/e2e/docker/calc-mcp/sam-node-config.yaml"
- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 2 20
+ mesh_wait_for_mcp_ready 2 30
local node2_peer_id
- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
+ node2_peer_id=$(mesh_node_peer_id 2)
+ [[ -n "${node2_peer_id}" ]]
echo "[$(date +%T)] Connecting Node 1 to Node 2"
- local node2_addr="/dns4/${MESH_PREFIX}-node-2/tcp/5002/p2p/${node2_peer_id}"
- run mesh_connect_peer 1 "${node2_addr}"
+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"
[[ "$status" -eq 0 ]]
mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20
diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bash
index e0a21e97..33cd2c9b 100644
--- a/tests/e2e/lib/container_mesh.bash
+++ b/tests/e2e/lib/container_mesh.bash
@@ -77,20 +77,29 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then
hexdump -vn 32 -e '1/1 "%02x"' /dev/urandom
}
- mesh_wait_for_log() {
- local container="$1"
- local needle="$2"
- local timeout_s="${3:-20}"
+ # mesh_wait_for_http waits until something answers HTTP at url on the test
+ # network. The status does not matter, a listener does.
+ mesh_wait_for_http() {
+ local url="$1"
+ local timeout_s="${2:-20}"
local i
- for ((i=0; i<timeout_s*10; i++)); do
- if docker logs "${container}" 2>&1 | grep -Fq "${needle}"; then
+ for ((i=0; i<timeout_s; i++)); do
+ if docker run --rm --network "${MESH_NETWORK}" "${MESH_RUNTIME_IMAGE}" \
+ curl -s -o /dev/null --max-time 5 "${url}"; then
return 0
fi
- sleep 0.1
+ sleep 1
done
+ echo "nothing answered at ${url} within ${timeout_s}s" >&2
return 1
}
+ # mesh_container_log prints a container's output, for failure messages only.
+ mesh_container_log() {
+ echo "--- ${1} ---" >&2
+ docker logs --tail "${2:-80}" "$1" >&2 2>&1 || true
+ }
+
# Pods matching a selector that will never become ready on their own.
# Deliberately excludes CrashLoopBackOff: the control plane restarts a few
# times while the database comes up, so treating that as terminal would swap
@@ -183,19 +192,63 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then
done
}
+ # mesh_wait_for_mcp_ready returns once node <idx> answers on its
+ # authenticated API, which the node serves only after it enrolled and a
+ # router admitted it. A node that exited first fails at once.
mesh_wait_for_mcp_ready() {
local idx="$1"
local timeout_s="${2:-20}"
+ local name="${MESH_PREFIX}-node-${idx}"
local i
for ((i=0; i<timeout_s; i++)); do
- if docker run --rm --network "${MESH_NETWORK}" python:3.12 curl -s -X POST -H "Content-Type: application/json" -H "X-Sam-Authentication: Bearer secret-token" -d '{"jsonrpc":"2.0","method":"ping","id":1}' --max-time 5 -D - http://${MESH_PREFIX}-node-${idx}:8080/mcp | grep -q "200 OK"; then
+ if [[ "$(docker inspect -f '{{.State.Running}}' "${name}" 2>/dev/null)" != "true" ]]; then
+ echo "${name} exited before serving its API" >&2
+ mesh_container_log "${name}"
+ return 1
+ fi
+ if docker run --rm --network "${MESH_NETWORK}" python:3.12 curl -s -X POST -H "Content-Type: application/json" -H "X-Sam-Authentication: Bearer secret-token" -d '{"jsonrpc":"2.0","method":"ping","id":1}' --max-time 5 -D - http://${name}:8080/mcp | grep -q "200 OK"; then
return 0
fi
sleep 1
done
+ echo "${name} did not serve its API within ${timeout_s}s" >&2
+ mesh_container_log "${name}"
return 1
}
+ # mesh_node_peer_id is node <idx>'s peer ID, as the node reports it.
+ mesh_node_peer_id() {
+ local idx="$1"
+ timeout 15s docker run --rm --network "${MESH_NETWORK}" "${MESH_RUNTIME_IMAGE}" \
+ mcp-client -url "http://${MESH_PREFIX}-node-${idx}:8080/mcp" -tool "get_mesh_info" 2>/dev/null |
+ jq -r '.peer_id // empty'
+ }
+
+ # mesh_node_addr is the address a peer dials node <idx> on within the test
+ # network.
+ mesh_node_addr() {
+ local idx="$1"
+ local peer_id
+ peer_id="$(mesh_node_peer_id "${idx}")"
+ [[ -n "${peer_id}" ]] || return 1
+ echo "/dns4/${MESH_PREFIX}-node-${idx}/tcp/5002/p2p/${peer_id}"
+ }
+
+ # mesh_admin_status is the control plane's view of the mesh, as it
+ # serialises it on /admin/status.
+ mesh_admin_status() {
+ docker run --rm --network "${MESH_NETWORK}" $(mesh_get_add_hosts) "${MESH_RUNTIME_IMAGE}" \
+ curl -sf --max-time 10 -H "Authorization: Bearer super-secret-admin-token" \
+ "http://sam-control-plane:8080/admin/status"
+ }
+
+ # mesh_enrolled_node prints the control plane's enrollment record for a
+ # peer, or nothing.
+ mesh_enrolled_node() {
+ local peer_id="$1"
+ mesh_admin_status | jq -c --arg id "${peer_id}" '.enrolled_nodes[] | select(.PeerID == $id)'
+ }
+
# POST /debug/connect-peer on node <idx>; the REST endpoint that replaced
# the connect_peer MCP tool (#318). Nonzero exit on any non-2xx response.
mesh_connect_peer() {
@@ -420,31 +473,26 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then
fi
mesh_wait_for_rollout statefulset/sam-router
- local i
- for ((i=0; i<200; i++)); do
- if kubectl --context="${KUBECONTEXT}" logs "sam-router-0" 2>&1 | grep -q "PeerID:"; then
- break
- fi
- sleep 0.1
- done
- local router_peer_id
- router_peer_id=$(kubectl --context="${KUBECONTEXT}" logs "sam-router-0" | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1 || true)
- [[ -n "${router_peer_id}" ]]
-
- # The router pod reports Ready before its lease reaches the control plane, and
- # a node's /register serves router addresses from that lease, so a node
- # started in between enrolls against an empty list and exits.
+ # The router pod reports Ready before its lease reaches the control plane,
+ # and a node's /register serves router addresses from that lease, so a
+ # node started in between enrolls against an empty list and exits. The
+ # lease is also where the router's peer ID comes from; on a reused
+ # cluster the freshest renewal is the router that is running now.
local router_node_ip
router_node_ip=$(docker inspect -f "{{(index .NetworkSettings.Networks \"${MESH_NETWORK:-kind}\").IPAddress}}" \
"$(kubectl --context="${KUBECONTEXT}" get pod sam-router-0 -o jsonpath='{.spec.nodeName}')")
+ local router_peer_id=""
local lease_deadline=$((SECONDS + 60))
- until docker run --rm --network "${MESH_NETWORK:-kind}" python:3.12 \
- curl -sf --max-time 5 "http://${router_node_ip}:8080/info" 2>/dev/null | grep -qaF "${router_peer_id}"; do
- if ((SECONDS >= lease_deadline)); then
+ while [[ -z "${router_peer_id}" ]]; do
+ router_peer_id=$(docker run --rm --network "${MESH_NETWORK:-kind}" "${MESH_RUNTIME_IMAGE}" \
+ curl -sf --max-time 5 -H "Authorization: Bearer super-secret-admin-token" \
+ "http://${router_node_ip}:8080/admin/status" 2>/dev/null |
+ jq -r '.active_routers | sort_by(.LastRenewal) | last | .PeerID // empty')
+ if [[ -z "${router_peer_id}" ]] && ((SECONDS >= lease_deadline)); then
echo "router lease did not reach the control plane within 60s" >&2
return 1
fi
- sleep 1
+ [[ -n "${router_peer_id}" ]] || sleep 1
done
echo "${router_peer_id}" > "/tmp/sam-wi-test-router-peer-id"
diff --git a/tests/e2e/policy.bats b/tests/e2e/policy.bats
index a9878964..c6572949 100644
--- a/tests/e2e/policy.bats
+++ b/tests/e2e/policy.bats
@@ -20,122 +20,9 @@ start_calc_mcp() {
--network-alias calc-mcp \
"${CALC_MCP_IMAGE}" >/dev/null
MESH_CONTAINERS+=("${name}")
- mesh_wait_for_log "${name}" "Uvicorn running on" 20
+ mesh_wait_for_http "http://calc-mcp:7777/mcp" 20
}
-# Custom mock OIDC server that returns 'data-scientist' role
-mesh_start_mock_oidc_custom() {
- local name="${MESH_PREFIX}-oidc"
- local cmd
- read -r -d '' cmd <<'EOF' || true
-python3 - <<'PY'
-import json
-import time
-import jwt
-from http.server import BaseHTTPRequestHandler, HTTPServer
-
-# The signing key is generated fresh on every start and the JWKS derived from
-# it, so no private key lives in the repository: an issuer whose key is public
-# would let anyone mint a token for any identity a control plane trusts.
-from cryptography.hazmat.primitives import serialization
-from cryptography.hazmat.primitives.asymmetric import rsa
-
-def _b64url_uint(n):
- import base64
- raw = n.to_bytes((n.bit_length() + 7) // 8, 'big')
- return base64.urlsafe_b64encode(raw).rstrip(b'=').decode('ascii')
-
-_KEY = rsa.generate_private_key(public_exponent=65537, key_size=2048)
-PRIVATE_KEY = _KEY.private_bytes(
- serialization.Encoding.PEM,
- serialization.PrivateFormat.PKCS8,
- serialization.NoEncryption(),
-)
-_PUB = _KEY.public_key().public_numbers()
-JWKS = {
- "keys": [
- {
- "kty": "RSA",
- "alg": "RS256",
- "use": "sig",
- "kid": "test-key-id",
- "n": _b64url_uint(_PUB.n),
- "e": _b64url_uint(_PUB.e),
- }
- ]
-}
-
-class Handler(BaseHTTPRequestHandler):
- def do_GET(self):
- if self.path == '/.well-known/openid-configuration':
- body = {
- 'issuer': 'http://mock-oidc:18080',
- 'authorization_endpoint': 'http://mock-oidc:18080/auth',
- 'token_endpoint': 'http://mock-oidc:18080/token',
- 'jwks_uri': 'http://mock-oidc:18080/keys'
- }
- data = json.dumps(body).encode('utf-8')
- self.send_response(200)
- self.send_header('Content-Type', 'application/json')
- self.send_header('Content-Length', str(len(data)))
- self.end_headers()
- self.wfile.write(data)
- return
- if self.path == '/keys':
- data = json.dumps(JWKS).encode('utf-8')
- self.send_response(200)
- self.send_header('Content-Type', 'application/json')
- self.send_header('Content-Length', str(len(data)))
- self.end_headers()
- self.wfile.write(data)
- return
- self.send_response(200)
- self.end_headers()
- self.wfile.write(b'ok')
-
- def do_POST(self):
- if self.path == '/token':
- payload = {
- 'iss': 'http://mock-oidc:18080',
- 'aud': 'sam-e2e',
- 'sub': 'test-user',
- 'exp': int(time.time()) + 3600,
- 'roles': ['data-scientist'] # Custom role
- }
- token = jwt.encode(payload, PRIVATE_KEY, algorithm='RS256', headers={'kid': 'test-key-id'})
- body = {
- 'access_token': token,
- 'token_type': 'Bearer',
- 'expires_in': 3600
- }
- data = json.dumps(body).encode('utf-8')
- self.send_response(200)
- self.send_header('Content-Type', 'application/json')
- self.send_header('Content-Length', str(len(data)))
- self.end_headers()
- self.wfile.write(data)
- return
- self.send_response(404)
- self.end_headers()
-
-print("Mock OIDC server ready", flush=True)
-HTTPServer(('0.0.0.0', 18080), Handler).serve_forever()
-PY
-EOF
-
- docker run -d \
- --name "${name}" \
- --network "${MESH_NETWORK}" \
- --network-alias mock-oidc \
- sam-mock-oidc:local \
- sh -c "${cmd}" >/dev/null
-
- MESH_CONTAINERS+=("${name}")
- mesh_wait_for_log "${name}" "Mock OIDC server ready" 30
-}
-
-
-
setup() {
mesh_setup_env
build_calc_mcp_image
@@ -166,43 +53,20 @@ attenuation:
# Start Node 1 (Target) with local policy file
mesh_start_node 1 "" "${config_file}"
- mesh_wait_for_log "${MESH_PREFIX}-node-1" "Successfully enrolled" 20
+ mesh_wait_for_mcp_ready 1 30
+
+ export TARGET_PEER_ID
+ TARGET_PEER_ID="$(mesh_node_peer_id 1)"
+ [[ -n "${TARGET_PEER_ID}" ]]
# Start Node 2 (Caller)
mesh_start_node 2
- mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 2
-
- local node2_id
- node2_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | grep -oE '12D3Koo[a-zA-Z0-9]+' | head -n 1)
-
- # Wait for discovery (Node 2 should see Node 1)
- local i
- local router_id
- router_id="$(cat "/tmp/${MESH_PREFIX}-router-peer-id")"
- export TARGET_PEER_ID=""
-
- for ((i=0; i<40; i++)); do
- local output
- output="$(docker run --rm --network "${MESH_NETWORK}" "${MESH_RUNTIME_IMAGE}" mcp-client -url "http://${MESH_PREFIX}-node-2:8080/mcp" -tool "get_mesh_info")"
- TARGET_PEER_ID=$(echo "${output}" | grep -oE '12D3Koo[a-zA-Z0-9]+' | grep -v "${router_id}" | grep -v "${node2_id}" | head -n 1)
- if [[ -n "${TARGET_PEER_ID}" ]]; then
- break
- fi
- sleep 1
- done
-
- echo "Node 2 logs after discovery loop:" >&3
- docker logs "${MESH_PREFIX}-node-2" >&3
-
- if [[ -z "${TARGET_PEER_ID}" ]]; then
- echo "Timeout waiting for discovery of Node 1"
- return 1
- fi
+ mesh_wait_for_mcp_ready 2 30
- # Explicitly connect Node 2 to Node 1 to avoid "no addresses" error
- local node1_addr="/dns4/${MESH_PREFIX}-node-1/tcp/5002/p2p/${TARGET_PEER_ID}"
- mesh_connect_peer 2 "${node1_addr}" >/dev/null
+ # Connect Node 2 to Node 1 explicitly so the calls below do not depend on
+ # discovery timing, and wait until Node 2 reports the connection.
+ mesh_connect_peer 2 "$(mesh_node_addr 1)" >/dev/null
+ mesh_wait_for_peer_connection 2 "${TARGET_PEER_ID}" 30
}
teardown() {
diff --git a/tests/e2e/python_sdk_test.bats b/tests/e2e/python_sdk_test.bats
index 36e7e22b..79cb7c5c 100644
--- a/tests/e2e/python_sdk_test.bats
+++ b/tests/e2e/python_sdk_test.bats
@@ -25,8 +25,7 @@ teardown() {
[[ "$status" -eq 0 ]]
local node1_name="${MESH_PREFIX}-node-1"
- mesh_wait_for_log "${node1_name}" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 1 20
+ mesh_wait_for_mcp_ready 1 30
# Use the Python SDK to interact with the node
run docker run --rm \
diff --git a/tests/e2e/relay.bats b/tests/e2e/relay.bats
index bed05813..8496ba7b 100644
--- a/tests/e2e/relay.bats
+++ b/tests/e2e/relay.bats
@@ -36,21 +36,6 @@ teardown() {
CLEANUP_NETWORKS=()
}
-@test "node starts with --enable-relay=true and logs message" {
- run mesh_start_mock_oidc
- [[ "$status" -eq 0 ]]
-
- run mesh_start_router
- [[ "$status" -eq 0 ]]
-
- # Start node with relay flag
- mesh_start_node "1" "--enable-relay=true --log-level=debug"
-
- # Wait for log message
- run mesh_wait_for_log "${MESH_PREFIX}-node-1" "Enabling Relay Service" 20
- [[ "$status" -eq 0 ]]
-}
-
@test "nodes in isolated networks discover and connect via relay" {
run mesh_start_mock_oidc
[[ "$status" -eq 0 ]]
@@ -123,23 +108,20 @@ EOF
# Gate on MCP actually answering: get_mesh_info polls below go through it, and
# a not-yet-listening sidecar burns discovery budget on empty responses.
- run mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 30
- [[ "$status" -eq 0 ]]
run mesh_wait_for_mcp_ready "1" 30
[[ "$status" -eq 0 ]]
+ local node1_peer_id node2_peer_id
+ node1_peer_id=$(mesh_node_peer_id 1)
+ [[ -n "${node1_peer_id}" ]]
OLD_NET=$MESH_NETWORK
MESH_NETWORK=$MESH_NETWORK_2
- run mesh_wait_for_log "${MESH_PREFIX}-node-2" "SAM Node Online" 30
- [[ "$status" -eq 0 ]]
run mesh_wait_for_mcp_ready "2" 30
[[ "$status" -eq 0 ]]
+ node2_peer_id=$(mesh_node_peer_id 2)
+ [[ -n "${node2_peer_id}" ]]
MESH_NETWORK=$OLD_NET
- local node1_peer_id node2_peer_id
- node1_peer_id=$(docker logs "${MESH_PREFIX}-node-1" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
- node2_peer_id=$(docker logs "${MESH_PREFIX}-node-2" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
-
# Assert on the specific peer, not on set size: a relayed connection can leave
# extra entries in connected_peers, so "length - 1" is not a stable count.
run mesh_wait_for_peer_connection "1" "${node2_peer_id}" 60
diff --git a/tests/e2e/services.bats b/tests/e2e/services.bats
index 58b44977..e47bac04 100644
--- a/tests/e2e/services.bats
+++ b/tests/e2e/services.bats
@@ -20,7 +20,7 @@ start_calc_mcp() {
--network-alias calc-mcp \
"${CALC_MCP_IMAGE}" >/dev/null
MESH_CONTAINERS+=("${name}")
- mesh_wait_for_log "${name}" "Uvicorn running on" 20
+ mesh_wait_for_http "http://calc-mcp:7777/mcp" 20
}
setup() {
@@ -41,8 +41,7 @@ teardown() {
echo "[$(date +%T)] Starting Node 1"
mesh_start_node 1 "--log-level debug"
local node1_name="${MESH_PREFIX}-node-1"
- mesh_wait_for_log "${node1_name}" "SAM Node Online" 60
- mesh_wait_for_mcp_ready 1 20
+ mesh_wait_for_mcp_ready 1 60
echo "[$(date +%T)] Starting calc-mcp backend"
start_calc_mcp
@@ -52,15 +51,14 @@ teardown() {
"--log-level debug" \
"tests/e2e/docker/calc-mcp/sam-node-config.yaml"
local node2_name="${MESH_PREFIX}-node-2"
- mesh_wait_for_log "${node2_name}" "SAM Node Online" 20
- mesh_wait_for_mcp_ready 2 20
+ mesh_wait_for_mcp_ready 2 30
local node2_peer_id
- node2_peer_id=$(docker logs "${node2_name}" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
+ node2_peer_id=$(mesh_node_peer_id 2)
+ [[ -n "${node2_peer_id}" ]]
echo "[$(date +%T)] Connecting Node 1 to Node 2"
- local node2_addr="/dns4/${node2_name}/tcp/5002/p2p/${node2_peer_id}"
- run mesh_connect_peer 1 "${node2_addr}"
+ run mesh_connect_peer 1 "$(mesh_node_addr 2)"
[[ "$status" -eq 0 ]]
mesh_wait_for_peer_connection 1 "${node2_peer_id}" 20
diff --git a/tests/e2e/standalone.bats b/tests/e2e/standalone.bats
index 22440142..dc3c7a27 100644
--- a/tests/e2e/standalone.bats
+++ b/tests/e2e/standalone.bats
@@ -51,6 +51,33 @@ wait_for_log() {
return 1
}
+# wait_for_node waits until the node named $1 serves its API on its socket,
+# which it does only after it enrolled, a router admitted it, and every
+# service in its configuration was registered. A node that exited fails at
+# once, with its log.
+wait_for_node() {
+ local name="$1" pid_var="NODE_${1^^}_PID"
+ local sock="$TEST_TMPDIR/node-$name/sam.sock"
+ for _ in $(seq 1 150); do
+ if ! kill -0 "${!pid_var}" 2>/dev/null; then
+ echo "node $name exited before serving its API:" >&2
+ cat "$TEST_TMPDIR/node-$name.log" >&2 || true
+ return 1
+ fi
+ curl -sf --unix-socket "$sock" http://localhost/healthz > /dev/null 2>&1 && return 0
+ sleep 0.2
+ done
+ echo "node $name did not serve its API in time:" >&2
+ cat "$TEST_TMPDIR/node-$name.log" >&2 || true
+ return 1
+}
+
+# node_peer_id is the peer ID node $1 reports for itself.
+node_peer_id() {
+ curl -sf --unix-socket "$TEST_TMPDIR/node-$1/sam.sock" http://localhost/debug/mesh-info |
+ python3 -c 'import json, sys; print(json.load(sys.stdin)["peer_id"])'
+}
+
# start_node boots a background sam-node joined via the bootstrap token; sets
# NODE_<NAME>_PID for teardown. The sidecar serves only on the Unix socket so
# two nodes on one host never fight over the default TCP bind, and loopback
@@ -117,18 +144,15 @@ EOF
start_node a --config "$TEST_TMPDIR/node-a-services.yaml"
start_node b
- wait_for_log "$TEST_TMPDIR/node-a.log" "SAM Node Online"
- wait_for_log "$TEST_TMPDIR/node-b.log" "SAM Node Online"
- peer_a="$(grep -oE 'PeerID: [A-Za-z0-9]+' "$TEST_TMPDIR/node-a.log" | head -1 | cut -d' ' -f2)"
+ wait_for_node a
+ wait_for_node b
+ peer_a="$(node_peer_id a)"
[[ -n "$peer_a" ]]
sock_a="$TEST_TMPDIR/node-a/sam.sock"
sock_b="$TEST_TMPDIR/node-b/sam.sock"
[[ -S "$sock_a" && -S "$sock_b" ]]
- # The mislabelled service is registered but withheld from discovery.
- wait_for_log "$TEST_TMPDIR/node-a.log" "not-an-mcp-server but not advertising it"
-
# Node B's OpenAI-compatible endpoint lists the model with A as its owner:
# the announcement crosses A -> router -> B over the mesh.
models=""
@@ -148,7 +172,9 @@ EOF
[[ "$status" -eq 0 ]]
[[ "$output" == *"hello from the mesh"* ]]
- # The plain HTTP server never shows up as an MCP provider.
+ # The plain HTTP server never shows up as an MCP provider: node A had
+ # registered every configured service before its API came up, so its
+ # absence here is a verdict, not a race.
run curl -sf --unix-socket "$sock_b" \
"http://localhost/sam/service/discover?type=mcp&name=not-an-mcp-server&timeout=3s"
[[ "$status" -eq 0 ]]