Skip to content

Agent Mesh

scale: member-journey harness, with an integration test and a CI job

Antonio Ojea3328d8e4d381
diff --git a/.github/workflows/member-journey.yaml b/.github/workflows/member-journey.yaml
new file mode 100644
index 00000000..a557529b
--- /dev/null
+++ b/.github/workflows/member-journey.yaml
@@ -0,0 +1,173 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: member-journey
+
+# Thirty members join one mesh at the same time from one host, then more join
+# behind them while the first ones stay resident. It is the join path under
+# concurrency, which no other suite exercises, and it keeps the harness that
+# runs the same journey at hundreds of members against a testnet
+# (tests/scale/member-journey.sh --env bananas) from rotting between
+# releases. A runner is a noisy place, so the latency bounds are loose; the
+# checks that gate are the ones that count: every member completed, nobody
+# dropped during the hold. Loopback is exempt from the routers' per-source
+# limits, so what this cannot see is what a shared public address sees.
+
+on:
+ push:
+ branches:
+ - "main"
+ workflow_dispatch:
+ inputs:
+ count:
+ description: "Members in the burst"
+ default: "30"
+ hold:
+ description: "How long the burst fleet stays resident"
+ default: "20s"
+
+permissions:
+ contents: read
+
+env:
+ GO_VERSION: "1.26"
+ CONTROL_PLANE: http://127.0.0.1:18090
+
+jobs:
+ member_journey:
+ runs-on: ubuntu-22.04
+ name: Member journey
+ timeout-minutes: 20
+ steps:
+ - name: Checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Setup Go
+ uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
+ with:
+ go-version: ${{ env.GO_VERSION }}
+ cache: false
+
+ - name: Build binaries
+ run: |
+ go build -o bin/sam-one ./cmd/sam-one
+ go build -o bin/sam-node ./cmd/sam-node
+ go build -o bin/sam-bench ./cmd/sam-bench
+ go build -o bin/stdio-mcp ./tests/scale/stdio-mcp
+
+ - name: Setup Python
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.12"
+
+ - name: Install the Python MCP SDK
+ run: python -m pip install --quiet "mcp>=2,<3"
+
+ - name: Start a local mesh
+ # sam-one is the control plane and the router on one port; its first
+ # boot writes the join token every member enrolls with.
+ run: |
+ mkdir -p "$RUNNER_TEMP/mesh"
+ bin/sam-one --data-dir "$RUNNER_TEMP/mesh" --port 18090 --bind-address 127.0.0.1 \
+ > "$RUNNER_TEMP/sam-one.log" 2>&1 &
+ for _ in $(seq 1 60); do
+ curl -fsS -o /dev/null "$CONTROL_PLANE/readyz" 2>/dev/null && break
+ sleep 1
+ done
+ curl -fsS -o /dev/null "$CONTROL_PLANE/readyz"
+ test -s "$RUNNER_TEMP/mesh/join-token"
+
+ - name: Start the providers
+ # Two nodes advertising the MCP service the members will look up and
+ # call, one backed by the Go SDK's stdio server and one by the Python
+ # SDK's, so a member's first call lands on a real server in either
+ # language and discovery has more than one record to return.
+ run: |
+ i=0
+ for backend in "$GITHUB_WORKSPACE/bin/stdio-mcp" "$(command -v python) $GITHUB_WORKSPACE/tests/scale/stdio-mcp.py"; do
+ i=$((i + 1))
+ dir="$RUNNER_TEMP/provider-$i"
+ mkdir -p "$dir"
+ command_json=$(python -c 'import json, sys; print(json.dumps(sys.argv[1].split()))' "$backend")
+ cat > "$dir/sam-node.yaml" <<EOF
+ version: "v1alpha1"
+ services:
+ - type: "mcp"
+ name: "everything"
+ description: "member-journey provider $i"
+ command: $command_json
+ EOF
+ bin/sam-node run \
+ --control-plane "$CONTROL_PLANE" \
+ --bootstrap-token-path "$RUNNER_TEMP/mesh/join-token" \
+ --config "$dir/sam-node.yaml" \
+ --data-dir "$dir" \
+ --bind-addr= \
+ --socket-path "$dir/node.sock" \
+ --metrics-addr "127.0.0.1:1909$i" \
+ --allow-loopback \
+ --listen /ip4/127.0.0.1/tcp/0 \
+ > "$RUNNER_TEMP/provider-$i.log" 2>&1 &
+ done
+ for i in 1 2; do
+ for _ in $(seq 1 60); do
+ [ "$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:1909$i/readyz")" = "200" ] && break
+ sleep 1
+ done
+ [ "$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:1909$i/readyz")" = "200" ]
+ grep -q 'Registered SERVICE_TYPE_MCP/everything' "$RUNNER_TEMP/provider-$i.log"
+ done
+
+ - name: Run the member journey
+ env:
+ COUNT: ${{ inputs.count || '30' }}
+ HOLD: ${{ inputs.hold || '20s' }}
+ # Loose on purpose: a runner's latency says nothing about the mesh.
+ T_JOURNEY_P50_MS: "20000"
+ T_JOURNEY_P95_MS: "60000"
+ run: |
+ tests/scale/member-journey.sh \
+ --control-plane "$CONTROL_PLANE" \
+ --bootstrap-token-path "$RUNNER_TEMP/mesh/join-token" \
+ --node-arg=--allow-loopback \
+ --node-arg=--listen=/ip4/127.0.0.1/tcp/0 \
+ --count "$COUNT" --late 5 --ramp 10 --hold "$HOLD" \
+ --out "$RUNNER_TEMP/journey" 2>&1 | tee "$RUNNER_TEMP/journey.out"
+ test "${PIPESTATUS[0]}" -eq 0
+
+ - name: Publish the table
+ if: always()
+ run: |
+ {
+ echo "## Member journey"
+ echo
+ sed -n '/^| phase/,$p' "$RUNNER_TEMP/journey.out" 2>/dev/null || echo "no summary was produced"
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Collect observations and logs
+ if: always()
+ run: |
+ mkdir -p _artifacts
+ cp "$RUNNER_TEMP"/journey/*.json "$RUNNER_TEMP"/journey/*.log "$RUNNER_TEMP"/journey/*.jsonl _artifacts/ 2>/dev/null || true
+ cp "$RUNNER_TEMP"/sam-one.log "$RUNNER_TEMP"/provider-*.log _artifacts/ 2>/dev/null || true
+
+ - name: Upload observations
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: member-journey-${{ github.run_id }}
+ path: _artifacts/
+ retention-days: 14
diff --git a/tests/integration/member_burst_test.go b/tests/integration/member_burst_test.go
new file mode 100644
index 00000000..f3700a54
--- /dev/null
+++ b/tests/integration/member_burst_test.go
@@ -0,0 +1,178 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration_test
+
+import (
+ "context"
+ "fmt"
+ "net"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/sam/internal/bench"
+ "github.com/google/sam/internal/standalone"
+)
+
+// TestMemberBurstJoinsConcurrently pins the join path under concurrency, the
+// part of a new member's journey nothing else in the suite exercises: ten
+// nodes enroll from one bootstrap token at the same instant, each
+// authenticates to the router and calls a service another node advertises,
+// and all ten complete. It is the integration tier of
+// tests/scale/member-journey.sh, which takes the same journey at hundreds of
+// members against a testnet; this one guards the enrollment race and the
+// tool, not the latency.
+func TestMemberBurstJoinsConcurrently(t *testing.T) {
+ const members = 10
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
+
+ srv, err := standalone.New(standalone.Options{BindAddress: "127.0.0.1:0", DataDir: t.TempDir()})
+ if err != nil {
+ t.Fatalf("standalone.New: %v", err)
+ }
+ if err := srv.Start(ctx); err != nil {
+ t.Fatalf("standalone start: %v", err)
+ }
+ t.Cleanup(func() { _ = srv.Close() })
+ controlPlane := "http://" + srv.Addr()
+ _, portStr, _ := net.SplitHostPort(srv.Addr())
+ port, _ := strconv.Atoi(portStr)
+ waitForActiveRouters(t, port, 1, 10*time.Second)
+ joinToken := tokenPath(t, srv.JoinToken())
+
+ // The service the members call: a real MCP server, which the provider's
+ // advertisement probe needs before it publishes the record the members
+ // will look up.
+ backend := httptest.NewServer(newBoundaryMCPHandler(t))
+ defer backend.Close()
+
+ nodeBin := buildBinary(t, "./cmd/sam-node")
+ providerDir := t.TempDir()
+ provider := launchNode(t, nodeBin, os.Environ(), providerDir, "run",
+ "--control-plane", controlPlane,
+ "--data-dir", providerDir,
+ "--bootstrap-token-path", joinToken,
+ "--api-token-path", tokenPath(t, "test-token"),
+ "--allow-loopback",
+ "--config", writeNodeConfig(t, providerDir, nil, svcDecl{Type: "mcp", Name: "everything", TargetURL: backend.URL}),
+ )
+ provider.waitForAPI(t)
+
+ // A short path: a member's socket lives under its directory, and a Unix
+ // socket path has a hard limit.
+ fleet, err := os.MkdirTemp("", "burst-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.RemoveAll(fleet) })
+
+ report, err := bench.RunJoin(ctx, bench.JoinOptions{
+ NodeBin: nodeBin,
+ Count: members,
+ Dir: filepath.Join(fleet, "members"),
+ ControlPlane: controlPlane,
+ BootstrapTokenPath: joinToken,
+ Service: "everything",
+ ReadyTimeout: 30 * time.Second,
+ CallTimeout: 30 * time.Second,
+ MetricsBasePort: reservePortBlock(t, members),
+ NodeArgs: []string{"--allow-loopback"},
+ Log: testLogWriter{t},
+ })
+ if err != nil {
+ t.Fatalf("RunJoin: %v", err)
+ }
+
+ if report.Completed != members || report.Failed != 0 {
+ for _, m := range report.Members {
+ if m.Error == "" {
+ continue
+ }
+ t.Errorf("member %d: %s\n%s", m.Index, m.Error, logTail(filepath.Join(m.Dir, "node.log")))
+ }
+ t.Fatalf("completed %d of %d, failed %d: %v\nprovider log:\n%s", report.Completed, members, report.Failed, report.Errors, logTail(provider.logPath))
+ }
+ if report.ReadyMs.Count != members || report.JourneyMs.Count != members {
+ t.Errorf("ready/journey samples = %d/%d, want %d each", report.ReadyMs.Count, report.JourneyMs.Count, members)
+ }
+ for _, m := range report.Members {
+ if m.Provider != provider.peerID.String() {
+ t.Errorf("member %d reached %q, want the provider %s", m.Index, m.Provider, provider.peerID)
+ }
+ }
+
+ // Every member is a distinct enrollment the control plane holds: one
+ // token, consumed once per member, with no row lost to the race.
+ status := fetchAdminStatus(t, port, srv.AdminToken())
+ enrolled := 0
+ for _, n := range status.GetEnrolledNodes() {
+ if n.GetPeerId() != provider.peerID.String() && !strings.EqualFold(n.GetPeerId(), srv.PeerID()) {
+ enrolled++
+ }
+ }
+ if enrolled < members {
+ t.Errorf("control plane holds %d member enrollments, want %d", enrolled, members)
+ }
+ t.Logf("%d members: ready p50 %.0f ms p95 %.0f ms; journey p50 %.0f ms p95 %.0f ms max %.0f ms; %d call attempts",
+ members, report.ReadyMs.P50, report.ReadyMs.P95, report.JourneyMs.P50, report.JourneyMs.P95, report.JourneyMs.Max, report.CallAttempts)
+}
+
+// reservePortBlock finds n consecutive free loopback ports in the reserved
+// range and returns the first.
+func reservePortBlock(t *testing.T, n int) int {
+ t.Helper()
+ for range 50 {
+ base := getFreePort(t)
+ free := true
+ for i := 1; i < n; i++ {
+ l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i))
+ if err != nil {
+ free = false
+ break
+ }
+ _ = l.Close()
+ }
+ if free {
+ return base
+ }
+ }
+ t.Fatalf("no run of %d free ports", n)
+ return 0
+}
+
+// logTail returns the last part of a log file, for a failure message.
+func logTail(path string) string {
+ data, err := os.ReadFile(path) // #nosec G304 -- a log the test wrote
+ if err != nil {
+ return err.Error()
+ }
+ if len(data) > 3000 {
+ data = data[len(data)-3000:]
+ }
+ return string(data)
+}
+
+// testLogWriter routes a tool's progress lines into the test log.
+type testLogWriter struct{ t *testing.T }
+
+func (w testLogWriter) Write(p []byte) (int, error) {
+ w.t.Log(strings.TrimRight(string(p), "\n"))
+ return len(p), nil
+}
diff --git a/tests/scale/member-journey-summary.py b/tests/scale/member-journey-summary.py
new file mode 100644
index 00000000..3a8e33e1
--- /dev/null
+++ b/tests/scale/member-journey-summary.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+#
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Render a member-journey result directory as one table and a verdict.
+
+Usage: member-journey-summary.py DIR [JOURNEY_P50_MS JOURNEY_P95_MS OUTAGE_MAX_MS CP_P99_S]
+
+Exit status 1 when any check fails, so the harness and CI can use it as a gate.
+"""
+import glob
+import json
+import os
+import re
+import sys
+
+out = sys.argv[1]
+t_p50, t_p95, t_outage, t_cp_p99 = (float(v) for v in (sys.argv[2:6] + ["5000", "15000", "60000", "1"][len(sys.argv) - 2:]))
+
+verdicts = []
+
+
+def check(name, ok, detail):
+ verdicts.append((name, ok, detail))
+
+
+def load(phase):
+ path = os.path.join(out, phase + ".json")
+ return json.load(open(path))["join"] if os.path.exists(path) else None
+
+
+print()
+print("| phase | members | ready | completed | failed | ready p50/p95/max ms | journey p50/p95/max ms | calls/member |")
+print("| --- | --- | --- | --- | --- | --- | --- | --- |")
+left_behind = []
+for phase in ("burst", "late", "late-after-rollout"):
+ j = load(phase)
+ if not j:
+ continue
+ r, jo = j["ready_ms"], j["journey_ms"]
+ per = j["call_attempts"] / j["completed"] if j["completed"] else 0
+ print(f"| {phase} | {j['count']} | {j['ready']} | {j['completed']} | {j['failed']} | "
+ f"{r['p50']:.0f}/{r['p95']:.0f}/{r['max']:.0f} | {jo['p50']:.0f}/{jo['p95']:.0f}/{jo['max']:.0f} | {per:.2f} |")
+ for stage, n in sorted(j.get("errors", {}).items()):
+ left_behind.append(f"{phase}: {n} x {stage}")
+ for msg, n in sorted(j.get("call_errors", {}).items()):
+ left_behind.append(f"{phase}: {n} call attempt(s) failed with: {msg}")
+ check(f"{phase}: every member completed", j["failed"] == 0, f"{j['failed']} of {j['count']} failed")
+ if j["completed"]:
+ check(f"{phase}: journey p50 <= {t_p50:.0f} ms", jo["p50"] <= t_p50, f"{jo['p50']:.0f} ms")
+ check(f"{phase}: journey p95 <= {t_p95:.0f} ms", jo["p95"] <= t_p95, f"{jo['p95']:.0f} ms")
+for line_ in left_behind:
+ print(line_)
+
+burst = load("burst")
+h = burst.get("hold") if burst else None
+if h:
+ print()
+ print(f"hold: {h['seconds']:.0f} s, {h['samples']} samples; resident {h['resident']}, fewest ready {h['min_ready']}; "
+ f"{h['flaps']} flaps across {h['members_flapped']} members, outage p50/max "
+ f"{h['outage_ms']['p50']:.0f}/{h['outage_ms']['max']:.0f} ms; "
+ f"{h['members_not_ready_at_end']} not ready at the end, {h['members_exited']} exited")
+ check("hold: every member ready at the end", h["members_not_ready_at_end"] == 0, f"{h['members_not_ready_at_end']} not ready")
+ check("hold: no member process exited", h["members_exited"] == 0, f"{h['members_exited']} exited")
+ check(f"hold: longest outage <= {t_outage/1000:.0f} s", h["outage_ms"]["max"] <= t_outage, f"{h['outage_ms']['max']/1000:.1f} s")
+
+# What the mesh reported, as differences across the run.
+line = re.compile(r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)')
+
+
+def parse(path):
+ series = {}
+ if not os.path.exists(path):
+ return series
+ for raw in open(path):
+ m = line.match(raw)
+ if not m:
+ continue
+ labels = tuple(sorted(re.findall(r'(\w+)="((?:[^"\\]|\\.)*)"', m.group(2) or "")))
+ try:
+ series[(m.group(1), labels)] = float(m.group(3))
+ except ValueError:
+ pass
+ return series
+
+
+def parse_all(prefix, tag):
+ # One exposition per pod; counters are summed across pods with the pod
+ # name folded into the key, so a replica's own counter is only ever
+ # differenced against itself.
+ series = {}
+ for path in sorted(glob.glob(os.path.join(out, f"{prefix}-*-{tag}.prom"))):
+ pod = os.path.basename(path)[: -len(f"-{tag}.prom")]
+ for (name, labels), v in parse(path).items():
+ series[(name, labels + (("pod", pod),))] = v
+ return series
+
+
+def total(series, name, **match):
+ return sum(v for (n, labels), v in series.items()
+ if n == name and all(dict(labels).get(k) == val for k, val in match.items()))
+
+
+def delta_by(before, after, name, label):
+ keys = set()
+ for (n, labels) in list(before) + list(after):
+ if n == name:
+ keys.add(dict(labels).get(label))
+ return {k: round(total(after, name, **{label: k}) - total(before, name, **{label: k})) for k in sorted(keys, key=str)}
+
+
+def p99_of_delta(before, after, name):
+ # Histogram buckets are cumulative counts; the difference across the run
+ # is the run's own histogram, and p99 is the first bound holding 99%.
+ buckets = {}
+ for (n, labels), v in after.items():
+ if n == name + "_bucket":
+ le = dict(labels)["le"]
+ buckets[le] = buckets.get(le, 0) + v - before.get((n, labels), 0)
+ if not buckets or buckets.get("+Inf", 0) <= 0:
+ return None
+ target = 0.99 * buckets["+Inf"]
+ for le in sorted(buckets, key=lambda s: float(s)):
+ if buckets[le] >= target:
+ return float(le)
+ return float("inf")
+
+
+cp_before, cp_after = parse_all("sam-control-plane", "before"), parse_all("sam-control-plane", "after")
+if cp_before and cp_after:
+ print()
+ codes = delta_by(cp_before, cp_after, "sam_control_plane_http_requests_total", "code")
+ errors = sum(v for c, v in codes.items() if c and c.startswith("5"))
+ p99 = p99_of_delta(cp_before, cp_after, "sam_control_plane_http_request_duration_seconds")
+ # Both replicas export the same store-derived gauge; take one.
+ nodes = tuple(max([v for (n, labels), v in s.items() if n == "sam_control_plane_enrolled_nodes" and dict(labels).get("role") == "sam:role:node"] or [0])
+ for s in (cp_before, cp_after))
+ print(f"control plane: requests by code {codes}; p99 <= {p99} s; enrolled nodes {nodes[0]:.0f} -> {nodes[1]:.0f}")
+ check("control plane: no 5xx during the run", errors == 0, f"{errors:.0f} responses")
+ if p99 is not None:
+ check(f"control plane: request p99 <= {t_cp_p99:g} s", p99 <= t_cp_p99, f"<= {p99} s bucket")
+
+routers = sorted({re.sub(r"-(before|after|resident|rolled)\.prom$", "", os.path.basename(p))
+ for p in glob.glob(os.path.join(out, "sam-router-*.prom"))})
+refused_total = 0
+for r in routers:
+ b, a = parse(os.path.join(out, f"{r}-before.prom")), parse(os.path.join(out, f"{r}-after.prom"))
+ if not (b and a):
+ continue
+ refused = delta_by(b, a, "sam_router_inbound_connections_refused_total", "reason")
+ handshakes = delta_by(b, a, "sam_router_auth_handshakes_total", "result")
+ rss = (total(b, "process_resident_memory_bytes") / 2**20, total(a, "process_resident_memory_bytes") / 2**20)
+ peers = total(a, "sam_router_authenticated_peers")
+ refused_total += sum(refused.values())
+ print(f"{r}: refused {refused}; handshakes {handshakes}; rss {rss[0]:.0f} -> {rss[1]:.0f} MiB; authenticated peers now {peers:.0f}")
+if routers:
+ check("routers: no inbound connection refused", refused_total == 0,
+ f"{refused_total:.0f} refused (a per-source-IP limit shows here first)")
+
+print()
+failed = 0
+for name, ok, detail in verdicts:
+ print(f"{'PASS' if ok else 'FAIL'} {name} ({detail})")
+ failed += 0 if ok else 1
+print()
+print(f"{len(verdicts) - failed} of {len(verdicts)} checks passed; observations under {out}")
+sys.exit(1 if failed else 0)
diff --git a/tests/scale/member-journey.sh b/tests/scale/member-journey.sh
new file mode 100755
index 00000000..33baf86e
--- /dev/null
+++ b/tests/scale/member-journey.sh
@@ -0,0 +1,339 @@
+#!/usr/bin/env bash
+#
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# What does the Nth member experience?
+#
+# The density and fleet runs say how many agents one member can carry. This
+# run is about members: N sam-node processes join a real mesh from one host,
+# each is timed from start to its first call of a service through the mesh,
+# the fleet then stays resident while more members join and, if asked, while
+# the routers are restarted under it. Every member on this host shares one
+# source address, which is what a classroom or an office looks like to the
+# router.
+#
+# Phases, each written as its own observation under --out:
+# burst N members start at --ramp per second burst.json
+# late --late more join the populated mesh late.json
+# rollout (--rollout) restart the router StatefulSet, events.jsonl
+# then --late more join behind it late-after-rollout.json
+# hold the burst fleet stays for --hold, sampled burst.json (hold section)
+#
+# Then one table and a verdict against the thresholds below, which the
+# environment overrides (T_JOURNEY_P95_MS=20000 ...).
+#
+# Usage:
+# tests/scale/member-journey.sh --env bananas --count 500 --late 20 --hold 1h --rollout --out /var/log/journey
+# tests/scale/member-journey.sh --control-plane http://127.0.0.1:8080 --bootstrap-token-path ./join-token \
+# --node-arg=--insecure-control-plane --node-arg=--allow-loopback --count 20 --out /tmp/journey
+#
+# The members call an MCP service named --service, so some node on the mesh
+# has to advertise one; on a testnet that is the everything canary. For a
+# local mesh, tests/scale/stdio-mcp.py (Python SDK) and tests/scale/stdio-mcp
+# (Go SDK) are real servers a provider node can run as a command backend;
+# .github/workflows/member-journey.yaml shows the whole local setup.
+#
+# --env mints the bootstrap token from the control plane's admin secret and
+# snapshots the control plane's and routers' metrics around the run through
+# the API server. Without it, bring your own token; the verdict then covers
+# only what the members themselves saw.
+
+set -euo pipefail
+
+COUNT=50
+LATE=10
+RAMP=0
+HOLD=10m
+SERVICE=everything
+ENV_NAME=""
+CONTROL_PLANE=""
+BOOTSTRAP_TOKEN_PATH=""
+OUT=""
+ROLLOUT=0
+NODE_ARGS=()
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SAM_NODE="${SAM_NODE:-${REPO_ROOT}/bin/sam-node}"
+SAM_BENCH="${SAM_BENCH:-${REPO_ROOT}/bin/sam-bench}"
+# May carry arguments, e.g. KUBECTL="kubectl --context my-cluster".
+KUBECTL="${KUBECTL:-kubectl}"
+
+# Thresholds. The journey bound is the SDKs' dial timeout: a member that
+# takes longer than that would have failed as an SDK agent.
+T_JOURNEY_P50_MS="${T_JOURNEY_P50_MS:-5000}"
+T_JOURNEY_P95_MS="${T_JOURNEY_P95_MS:-15000}"
+T_OUTAGE_MAX_MS="${T_OUTAGE_MAX_MS:-60000}"
+T_CP_P99_S="${T_CP_P99_S:-1}"
+
+usage() {
+ sed -n '17,46p' "$0" | sed 's/^# \{0,1\}//'
+ cat <<EOF
+
+Options:
+ --env NAME testnet (hub, bananas): mints the token, snapshots metrics, enables --rollout
+ --control-plane URL control plane to enroll with (default https://NAME.sam-mesh.dev with --env)
+ --bootstrap-token-path F bootstrap token to enroll with; minted from the admin secret with --env if omitted
+ --count N members in the burst (default ${COUNT})
+ --late N members that join once the burst is resident, per late phase (default ${LATE}; 0 skips)
+ --ramp N members started per second (default ${RAMP}: all at once)
+ --hold DURATION how long the burst fleet stays resident (default ${HOLD})
+ --rollout restart the router StatefulSet while the fleet is resident (needs --env)
+ --service NAME MCP service each member finds and calls (default ${SERVICE})
+ --node-arg ARG extra sam-node argument, repeatable
+ --out DIR where observations, logs and member state land (required)
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --env) ENV_NAME="$2"; shift 2 ;;
+ --control-plane) CONTROL_PLANE="$2"; shift 2 ;;
+ --bootstrap-token-path) BOOTSTRAP_TOKEN_PATH="$2"; shift 2 ;;
+ --count) COUNT="$2"; shift 2 ;;
+ --late) LATE="$2"; shift 2 ;;
+ --ramp) RAMP="$2"; shift 2 ;;
+ --hold) HOLD="$2"; shift 2 ;;
+ --rollout) ROLLOUT=1; shift ;;
+ --service) SERVICE="$2"; shift 2 ;;
+ --node-arg) NODE_ARGS+=("$2"); shift 2 ;;
+ --node-arg=*) NODE_ARGS+=("${1#--node-arg=}"); shift ;;
+ --out) OUT="$2"; shift 2 ;;
+ -h|--help) usage; exit 0 ;;
+ *) echo "unknown flag: $1" >&2; usage >&2; exit 2 ;;
+ esac
+done
+
+fail() { echo "member-journey: $*" >&2; exit 1; }
+log() { echo "$(date -u +%FT%TZ) $*"; }
+
+[[ -n "$OUT" ]] || fail "--out is required"
+[[ -x "$SAM_NODE" ]] || fail "sam-node not found at $SAM_NODE; run make or set SAM_NODE"
+[[ -x "$SAM_BENCH" ]] || fail "sam-bench not found at $SAM_BENCH; run make or set SAM_BENCH"
+command -v python3 >/dev/null || fail "python3 is needed for the summary"
+if [[ -n "$ENV_NAME" ]]; then
+ NS="sam-${ENV_NAME}"
+ command -v "${KUBECTL%% *}" >/dev/null || fail "kubectl is needed with --env"
+ $KUBECTL get namespace "$NS" >/dev/null 2>&1 || fail "namespace $NS is not reachable with the current kubectl context"
+ CONTROL_PLANE="${CONTROL_PLANE:-https://${ENV_NAME}.sam-mesh.dev}"
+else
+ [[ -n "$CONTROL_PLANE" ]] || fail "--control-plane is required without --env"
+ [[ -n "$BOOTSTRAP_TOKEN_PATH" ]] || fail "--bootstrap-token-path is required without --env"
+ [[ "$ROLLOUT" -eq 0 ]] || fail "--rollout needs --env"
+fi
+
+mkdir -p "$OUT"
+for phase in burst late late-after-rollout; do
+ [[ -e "$OUT/$phase" ]] && fail "$OUT/$phase exists; a member directory with an identity would resume rather than join. Use a fresh --out"
+done
+
+# Every member is a process with its own sockets and a libp2p host. The soft
+# limit is per process, so this only has to cover one node, but it is often 1024.
+ulimit -n 65536 2>/dev/null || log "could not raise the file descriptor limit; a member may run out"
+
+JOIN_PID=""
+PF_PID=""
+TOKEN_ID=""
+SUMMARIZED=0
+cleanup() {
+ local status=$?
+ if [[ -n "$JOIN_PID" ]] && kill -0 "$JOIN_PID" 2>/dev/null; then
+ # SIGINT ends the hold; the run still writes its report and stops its members.
+ kill -INT "$JOIN_PID" 2>/dev/null || true
+ wait "$JOIN_PID" 2>/dev/null || true
+ fi
+ JOIN_PID=""
+ [[ -n "$PF_PID" ]] && kill "$PF_PID" 2>/dev/null || true
+ PF_PID=""
+ if [[ "$SUMMARIZED" -eq 0 && -f "$OUT/burst.json" ]]; then
+ summarize || true
+ fi
+ [[ -n "$TOKEN_ID" ]] && revoke_token
+ rm -f "$OUT/admin-header"
+ exit "$status"
+}
+trap cleanup EXIT
+
+# admin_api runs curl against the control plane's admin API over a
+# port-forward, with the admin token in a header file rather than on a
+# command line. Arguments are curl's, after the method and path.
+admin_api() {
+ local method="$1" path="$2"
+ shift 2
+ local port
+ port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])')
+ if [[ ! -s "$OUT/admin-header" ]]; then
+ $KUBECTL get secret "sam-control-plane-secret-${ENV_NAME}" -n "$NS" -o jsonpath='{.data.admin-token}' \
+ | base64 -d | sed 's/^/Authorization: Bearer /' > "$OUT/admin-header"
+ chmod 600 "$OUT/admin-header"
+ fi
+ $KUBECTL port-forward "deployment/sam-control-plane-${ENV_NAME}" -n "$NS" "${port}:8080" >/dev/null 2>&1 &
+ PF_PID=$!
+ for _ in $(seq 1 30); do
+ curl -fsS -o /dev/null "http://127.0.0.1:${port}/healthz" 2>/dev/null && break
+ sleep 0.5
+ done
+ local status=0
+ curl -fsS -X "$method" -H "@${OUT}/admin-header" "$@" "http://127.0.0.1:${port}${path}" || status=$?
+ kill "$PF_PID" 2>/dev/null || true
+ PF_PID=""
+ return "$status"
+}
+
+# mint_token writes a bootstrap token with enough usages for every phase and
+# remembers its id so the run can revoke it on the way out.
+mint_token() {
+ local uses=$(( COUNT + 2 * LATE + 5 ))
+ admin_api POST /admin/bootstrap-tokens -H 'Content-Type: application/json' \
+ -d "{\"role\":\"sam:role:node\",\"max_usages\":${uses},\"ttl_hours\":48,\"description\":\"member-journey ${COUNT}+${LATE} from $(hostname)\"}" \
+ > "$OUT/token.json" || fail "minting a bootstrap token failed"
+ python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["token"])' "$OUT/token.json" > "$OUT/bootstrap-token"
+ TOKEN_ID=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["id"])' "$OUT/token.json")
+ rm -f "$OUT/token.json"
+ chmod 600 "$OUT/bootstrap-token"
+ [[ -s "$OUT/bootstrap-token" && -n "$TOKEN_ID" ]] || fail "no bootstrap token came back"
+ BOOTSTRAP_TOKEN_PATH="$OUT/bootstrap-token"
+ log "minted bootstrap token ${TOKEN_ID} for ${uses} enrollments"
+}
+
+# revoke_token retires the token a run minted: a test must not leave a
+# standing credential on a public testnet.
+revoke_token() {
+ if admin_api DELETE "/admin/bootstrap-tokens/${TOKEN_ID}" -o /dev/null; then
+ log "revoked bootstrap token ${TOKEN_ID}"
+ else
+ log "could not revoke bootstrap token ${TOKEN_ID}; revoke it by hand"
+ fi
+ TOKEN_ID=""
+ rm -f "$OUT/bootstrap-token"
+}
+
+# snapshot reads what every control plane and router pod exports, through
+# the API server so no port is opened for it. Per pod, not per Service: the
+# control plane has two replicas and a Service proxy alternates between
+# them, which would make a before/after difference meaningless. The
+# members' own numbers say what they experienced; these say what the mesh
+# did to produce it.
+snapshot() {
+ local tag="$1" pod name
+ [[ -n "$ENV_NAME" ]] || return 0
+ for pod in $($KUBECTL get pods -n "$NS" -l "app=sam-control-plane-${ENV_NAME}" -o name 2>/dev/null); do
+ name="${pod#pod/}"
+ $KUBECTL get --raw "/api/v1/namespaces/${NS}/pods/${name}:8080/proxy/metrics" \
+ > "$OUT/${name}-${tag}.prom" 2>/dev/null || log "could not read ${name} metrics (${tag})"
+ done
+ for pod in $($KUBECTL get pods -n "$NS" -l "app=sam-router-${ENV_NAME}" -o name 2>/dev/null); do
+ name="${pod#pod/}"
+ $KUBECTL get --raw "/api/v1/namespaces/${NS}/pods/${name}:9090/proxy/metrics" \
+ > "$OUT/${name}-${tag}.prom" 2>/dev/null || log "could not read ${name} metrics (${tag})"
+ done
+}
+
+event() {
+ printf '{"event":"%s","at":"%s"}\n' "$1" "$(date -u +%FT%TZ)" >> "$OUT/events.jsonl"
+}
+
+# summarize closes the run: a last snapshot, then the table and verdict. It
+# runs from the exit trap too, so an interrupted hold still gets its report.
+summarize() {
+ SUMMARIZED=1
+ event hold-end
+ snapshot after
+ python3 "${REPO_ROOT}/tests/scale/member-journey-summary.py" "$OUT" \
+ "$T_JOURNEY_P50_MS" "$T_JOURNEY_P95_MS" "$T_OUTAGE_MAX_MS" "$T_CP_P99_S"
+}
+
+# join_cmd fills JOIN_CMD with one phase's sam-bench invocation. It is an
+# array rather than a function so the burst can be backgrounded directly:
+# a backgrounded function is a subshell, and a signal to it would not reach
+# the run inside.
+JOIN_CMD=()
+join_cmd() { # phase count base-port extra...
+ local phase="$1" count="$2" base="$3" a
+ shift 3
+ JOIN_CMD=("$SAM_BENCH" join
+ --node-bin "$SAM_NODE"
+ --control-plane "$CONTROL_PLANE"
+ --bootstrap-token-path "$BOOTSTRAP_TOKEN_PATH"
+ --service "$SERVICE"
+ --count "$count"
+ --metrics-base-port "$base"
+ --dir "$OUT/$phase"
+ --label "phase=$phase" --label "count=$count" --label "burst=$COUNT" --label "host=$(hostname)"
+ --out "$OUT/$phase.json")
+ for a in "${NODE_ARGS[@]}"; do JOIN_CMD+=("--node-arg=$a"); done
+ JOIN_CMD+=("$@")
+}
+
+[[ -n "$BOOTSTRAP_TOKEN_PATH" ]] || mint_token
+rm -f "$OUT/resident"
+: > "$OUT/events.jsonl"
+{
+ echo "control_plane=$CONTROL_PLANE"
+ echo "host=$(hostname) nproc=$(nproc) mem_kib=$(awk '/MemTotal/{print $2}' /proc/meminfo)"
+ echo "sam_node=$("$SAM_NODE" --version 2>/dev/null | head -1)"
+} > "$OUT/environment.txt"
+
+snapshot before
+event burst-start
+log "burst: ${COUNT} members at ${RAMP}/s, then holding ${HOLD}"
+join_cmd burst "$COUNT" 20000 --ramp "$RAMP" --hold "$HOLD" --resident-marker "$OUT/resident"
+"${JOIN_CMD[@]}" 2> "$OUT/burst.log" &
+JOIN_PID=$!
+while [[ ! -f "$OUT/resident" ]]; do
+ kill -0 "$JOIN_PID" 2>/dev/null || fail "the burst ended before the fleet was resident; see $OUT/burst.log"
+ sleep 2
+done
+event burst-resident
+snapshot resident
+python3 - "$OUT/burst.json" <<'PY'
+import json, sys
+j = json.load(open(sys.argv[1]))["join"]
+print(f"burst: {j['ready']}/{j['count']} ready, {j['completed']} completed, {j['failed']} failed; "
+ f"journey p50 {j['journey_ms']['p50']:.0f} ms, p95 {j['journey_ms']['p95']:.0f} ms, max {j['journey_ms']['max']:.0f} ms")
+for stage, n in sorted(j.get("errors", {}).items()):
+ print(f" {n} x {stage}")
+PY
+
+if (( LATE > 0 )); then
+ event late-start
+ log "late: ${LATE} members join the populated mesh"
+ join_cmd late "$LATE" 30000
+ "${JOIN_CMD[@]}" 2> "$OUT/late.log" || log "late phase did not run cleanly; see $OUT/late.log"
+ event late-end
+fi
+
+if (( ROLLOUT )); then
+ event rollout-start
+ log "rollout: restarting statefulset/sam-router-${ENV_NAME} under the resident fleet"
+ $KUBECTL rollout restart "statefulset/sam-router-${ENV_NAME}" -n "$NS"
+ $KUBECTL rollout status "statefulset/sam-router-${ENV_NAME}" -n "$NS" --timeout=600s || log "rollout did not finish in 600s"
+ event rollout-end
+ snapshot rolled
+ if (( LATE > 0 )); then
+ # The routers' addresses just changed and the DNS records catch up on a
+ # timer; a long bound here measures that wait instead of hiding it.
+ event late-after-rollout-start
+ log "late: ${LATE} members join behind the rollout"
+ join_cmd late-after-rollout "$LATE" 31000 --ready-timeout 600s
+ "${JOIN_CMD[@]}" 2> "$OUT/late-after-rollout.log" \
+ || log "late-after-rollout phase did not run cleanly; see $OUT/late-after-rollout.log"
+ event late-after-rollout-end
+ fi
+fi
+
+log "holding; the burst fleet stays until ${HOLD} after it became resident (Ctrl-C ends it early with a report)"
+wait "$JOIN_PID" || log "burst run exited non-zero; see $OUT/burst.log"
+JOIN_PID=""
+summarize
diff --git a/tests/scale/stdio-mcp.py b/tests/scale/stdio-mcp.py
new file mode 100644
index 00000000..39f4f00e
--- /dev/null
+++ b/tests/scale/stdio-mcp.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+#
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""An MCP server over stdio, for a provider node in a local mesh.
+
+Built on the official Python SDK (the `mcp` package, as in
+development/examples), so what a member reaches through the mesh is a real
+server and not an imitation of one. A node declares it as a command backend
+and advertises it; the member-journey harness calls it as "everything".
+
+ pip install "mcp>=2,<3"
+
+ services:
+ - type: mcp
+ name: everything
+ command: ["python3", "tests/scale/stdio-mcp.py"]
+"""
+from mcp.server.mcpserver import MCPServer
+
+mcp = MCPServer("everything")
+
+
+def echo(text: str) -> str:
+ """Return the text given."""
+ return text
+
+
+if __name__ == "__main__":
+ mcp.run(transport="stdio")
diff --git a/tests/scale/stdio-mcp/main.go b/tests/scale/stdio-mcp/main.go
new file mode 100644
index 00000000..75d0c093
--- /dev/null
+++ b/tests/scale/stdio-mcp/main.go
@@ -0,0 +1,50 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Command stdio-mcp is an MCP server over stdio, for a provider node in a
+// local mesh. It is the Go twin of tests/scale/stdio-mcp.py, built on the
+// official Go SDK, so a member's first call through the mesh lands on a real
+// server in either language.
+//
+// go build -o bin/stdio-mcp ./tests/scale/stdio-mcp
+//
+// services:
+// - type: mcp
+// name: everything
+// command: ["bin/stdio-mcp"]
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/modelcontextprotocol/go-sdk/mcp"
+)
+
+type echoInput struct {
+ Text string `json:"text" jsonschema:"the text to return"`
+}
+
+func main() {
+ server := mcp.NewServer(&mcp.Implementation{Name: "everything", Version: "0"}, nil)
+ mcp.AddTool(server, &mcp.Tool{Name: "echo", Description: "Return the text given"},
+ func(_ context.Context, _ *mcp.CallToolRequest, in echoInput) (*mcp.CallToolResult, any, error) {
+ return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: in.Text}}}, nil, nil
+ })
+ if err := server.Run(context.Background(), &mcp.StdioTransport{}); err != nil {
+ fmt.Fprintln(os.Stderr, "stdio-mcp:", err)
+ os.Exit(1)
+ }
+}