Skip to content

Agent Mesh

deploy: let the probe try every provider it discovers

Antonio Ojea3eb450310ae1
diff --git a/.github/k8s/sam-probe-cronjob-template.yaml b/.github/k8s/sam-probe-cronjob-template.yaml
index e4ded877..b3053864 100644
--- a/.github/k8s/sam-probe-cronjob-template.yaml
+++ b/.github/k8s/sam-probe-cronjob-template.yaml
@@ -19,7 +19,8 @@ spec:
# One attempt: a retry would hide exactly the flakiness this exists to
# measure. A failed Job is the signal.
backoffLimit: 0
- activeDeadlineSeconds: 300
+ # 180s to be ready plus 120s to reach a provider, with headroom.
+ activeDeadlineSeconds: 360
template:
metadata:
labels:
@@ -118,35 +119,46 @@ spec:
ROUTER_MS=$(printf '%s' "$CONN" | grep -oE '"router_latency_ms":[0-9]+' | grep -oE '[0-9]+$')
PEERS=$(printf '%s' "$CONN" | grep -oE '"connected_peers":[0-9]+' | grep -oE '[0-9]+$')
- # 3. A service someone else advertised is discoverable. A node
- # that joined seconds ago has a thin routing table, so the lookup
- # is retried for a while; how long it takes is itself reported.
+ # 3. A service someone else advertised is discoverable, and one
+ # of its providers answers an MCP initialize through the mesh.
+ # Discovery is best-effort per peer: a provider record can name a
+ # pod that a rollout just replaced, and a node that joined
+ # seconds ago has a thin routing table. So every provider is
+ # tried, the whole thing is retried for a while, and how long it
+ # took is itself reported. No trailing slash on the call: the
+ # path maps onto the service's target_url as-is. The body is JSON
+ # or an SSE frame; either names the server.
PEER=""
- DISCOVER_START=$(date +%s)
+ TRIED=0
+ LAST_ERR=""
+ REACH_START=$(date +%s)
while [ -z "$PEER" ]; do
if curl -sf --unix-socket "$SOCK" -o /tmp/providers.json \
"http://localhost/sam/service/discover?type=mcp&name=${PROBE_SERVICE}&timeout=20s"; then
- PEER=$(grep -oE '"peer_id":"[^"]+"' /tmp/providers.json | head -1 | cut -d'"' -f4)
+ for p in $(grep -oE '"peer_id":"[^"]+"' /tmp/providers.json | cut -d'"' -f4); do
+ TRIED=$((TRIED + 1))
+ set -- $(curl -s --unix-socket "$SOCK" -o /tmp/init.out -w '%{http_code} %{time_total}' \
+ -X POST "http://localhost/sam/${p}/mcp/${PROBE_SERVICE}" \
+ -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' \
+ -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"sam-probe","version":"0"}}}')
+ # curl prints "000 0" when it cannot connect; the defaults
+ # cover curl printing nothing at all, so set -u cannot end
+ # the run without a verdict line.
+ if [ "${1:-}" = "200" ] && grep -q '"serverInfo"' /tmp/init.out; then
+ PEER=$p; CALL_S=${2:-0}
+ break
+ fi
+ LAST_ERR="${p}: HTTP ${1:-000} $(head -c 120 /tmp/init.out | tr -d '"\n')"
+ done
fi
[ -n "$PEER" ] && break
- [ $(( $(date +%s) - DISCOVER_START )) -ge 90 ] && fail discover "no provider advertises ${PROBE_SERVICE} after 90s"
- sleep 3
+ [ $(( $(date +%s) - REACH_START )) -ge 120 ] && fail call "no provider of ${PROBE_SERVICE} answered initialize in 120s (${TRIED} attempts; last: ${LAST_ERR:-none discovered})"
+ sleep 5
done
- DISCOVER_S=$(( $(date +%s) - DISCOVER_START ))
+ REACH_S=$(( $(date +%s) - REACH_START ))
- # 4. And answers an MCP initialize through the mesh, end to end.
- # No trailing slash: the path maps onto the service's target_url
- # as-is. The body is JSON or an SSE frame; either names the server.
- set -- $(curl -s --unix-socket "$SOCK" -o /tmp/init.out -w '%{http_code} %{time_total}' \
- -X POST "http://localhost/sam/${PEER}/mcp/${PROBE_SERVICE}" \
- -H 'Content-Type: application/json' -H 'Accept: application/json, text/event-stream' \
- -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"sam-probe","version":"0"}}}')
- CALL_CODE=$1; CALL_S=$2
- [ "$CALL_CODE" = "200" ] || fail call "initialize on ${PEER} returned HTTP ${CALL_CODE}: $(head -c 200 /tmp/init.out)"
- grep -q '"serverInfo"' /tmp/init.out || fail call "initialize on ${PEER} returned no serverInfo: $(head -c 200 /tmp/init.out)"
-
- printf '{"probe":"sam-cold-path","ok":true,"ready_s":%d,"router_latency_ms":%s,"connected_peers":%s,"discover_s":%d,"call_s":%s,"provider":"%s","elapsed_s":%d}\n' \
- "$READY_S" "${ROUTER_MS:-null}" "${PEERS:-null}" "$DISCOVER_S" "$CALL_S" "$PEER" "$(( $(date +%s) - START ))"
+ printf '{"probe":"sam-cold-path","ok":true,"ready_s":%d,"router_latency_ms":%s,"connected_peers":%s,"reach_s":%d,"providers_tried":%d,"call_s":%s,"provider":"%s","elapsed_s":%d}\n' \
+ "$READY_S" "${ROUTER_MS:-null}" "${PEERS:-null}" "$REACH_S" "$TRIED" "$CALL_S" "$PEER" "$(( $(date +%s) - START ))"
volumes:
- name: config-volume
configMap:
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
index 15ead65a..d4253f06 100644
--- a/.github/workflows/deploy.yaml
+++ b/.github/workflows/deploy.yaml
@@ -809,7 +809,7 @@ jobs:
# `kubectl wait` can only wait for one condition; a failed Job would
# otherwise sit out the whole timeout.
- for _ in $(seq 1 60); do
+ for _ in $(seq 1 72); do
status=$(kubectl get job "${JOB}" -n "${CANARY_NAMESPACE}" -o jsonpath='{range .status.conditions[*]}{.type}={.status}{"\n"}{end}')
case "${status}" in
*Complete=True*) break ;;