Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
scale: member-journey harness, with an integration test and a CI job
Antonio Ojea3328d8e4d381
diff --git a/.github/workflows/member-journey.yaml b/.github/workflows/member-journey.yamlnew file mode 100644index 00000000..a557529b--- /dev/null+++ b/.github/workflows/member-journey.yaml@@ -0,0 +1,173 @@+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++name: member-journey++# Thirty members join one mesh at the same time from one host, then more join+# behind them while the first ones stay resident. It is the join path under+# concurrency, which no other suite exercises, and it keeps the harness that+# runs the same journey at hundreds of members against a testnet+# (tests/scale/member-journey.sh --env bananas) from rotting between+# releases. A runner is a noisy place, so the latency bounds are loose; the+# checks that gate are the ones that count: every member completed, nobody+# dropped during the hold. Loopback is exempt from the routers' per-source+# limits, so what this cannot see is what a shared public address sees.++on:+ push:+ branches:+ - "main"+ workflow_dispatch:+ inputs:+ count:+ description: "Members in the burst"+ default: "30"+ hold:+ description: "How long the burst fleet stays resident"+ default: "20s"++permissions:+ contents: read++env:+ GO_VERSION: "1.26"+ CONTROL_PLANE: http://127.0.0.1:18090++jobs:+ member_journey:+ runs-on: ubuntu-22.04+ name: Member journey+ timeout-minutes: 20+ steps:+ - name: Checkout+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ - name: Setup Go+ uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5+ with:+ go-version: ${{ env.GO_VERSION }}+ cache: false++ - name: Build binaries+ run: |+ go build -o bin/sam-one ./cmd/sam-one+ go build -o bin/sam-node ./cmd/sam-node+ go build -o bin/sam-bench ./cmd/sam-bench+ go build -o bin/stdio-mcp ./tests/scale/stdio-mcp++ - name: Setup Python+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: "3.12"++ - name: Install the Python MCP SDK+ run: python -m pip install --quiet "mcp>=2,<3"++ - name: Start a local mesh+ # sam-one is the control plane and the router on one port; its first+ # boot writes the join token every member enrolls with.+ run: |+ mkdir -p "$RUNNER_TEMP/mesh"+ bin/sam-one --data-dir "$RUNNER_TEMP/mesh" --port 18090 --bind-address 127.0.0.1 \+ > "$RUNNER_TEMP/sam-one.log" 2>&1 &+ for _ in $(seq 1 60); do+ curl -fsS -o /dev/null "$CONTROL_PLANE/readyz" 2>/dev/null && break+ sleep 1+ done+ curl -fsS -o /dev/null "$CONTROL_PLANE/readyz"+ test -s "$RUNNER_TEMP/mesh/join-token"++ - name: Start the providers+ # Two nodes advertising the MCP service the members will look up and+ # call, one backed by the Go SDK's stdio server and one by the Python+ # SDK's, so a member's first call lands on a real server in either+ # language and discovery has more than one record to return.+ run: |+ i=0+ for backend in "$GITHUB_WORKSPACE/bin/stdio-mcp" "$(command -v python) $GITHUB_WORKSPACE/tests/scale/stdio-mcp.py"; do+ i=$((i + 1))+ dir="$RUNNER_TEMP/provider-$i"+ mkdir -p "$dir"+ command_json=$(python -c 'import json, sys; print(json.dumps(sys.argv[1].split()))' "$backend")+ cat > "$dir/sam-node.yaml" <<EOF+ version: "v1alpha1"+ services:+ - type: "mcp"+ name: "everything"+ description: "member-journey provider $i"+ command: $command_json+ EOF+ bin/sam-node run \+ --control-plane "$CONTROL_PLANE" \+ --bootstrap-token-path "$RUNNER_TEMP/mesh/join-token" \+ --config "$dir/sam-node.yaml" \+ --data-dir "$dir" \+ --bind-addr= \+ --socket-path "$dir/node.sock" \+ --metrics-addr "127.0.0.1:1909$i" \+ --allow-loopback \+ --listen /ip4/127.0.0.1/tcp/0 \+ > "$RUNNER_TEMP/provider-$i.log" 2>&1 &+ done+ for i in 1 2; do+ for _ in $(seq 1 60); do+ [ "$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:1909$i/readyz")" = "200" ] && break+ sleep 1+ done+ [ "$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:1909$i/readyz")" = "200" ]+ grep -q 'Registered SERVICE_TYPE_MCP/everything' "$RUNNER_TEMP/provider-$i.log"+ done++ - name: Run the member journey+ env:+ COUNT: ${{ inputs.count || '30' }}+ HOLD: ${{ inputs.hold || '20s' }}+ # Loose on purpose: a runner's latency says nothing about the mesh.+ T_JOURNEY_P50_MS: "20000"+ T_JOURNEY_P95_MS: "60000"+ run: |+ tests/scale/member-journey.sh \+ --control-plane "$CONTROL_PLANE" \+ --bootstrap-token-path "$RUNNER_TEMP/mesh/join-token" \+ --node-arg=--allow-loopback \+ --node-arg=--listen=/ip4/127.0.0.1/tcp/0 \+ --count "$COUNT" --late 5 --ramp 10 --hold "$HOLD" \+ --out "$RUNNER_TEMP/journey" 2>&1 | tee "$RUNNER_TEMP/journey.out"+ test "${PIPESTATUS[0]}" -eq 0++ - name: Publish the table+ if: always()+ run: |+ {+ echo "## Member journey"+ echo+ sed -n '/^| phase/,$p' "$RUNNER_TEMP/journey.out" 2>/dev/null || echo "no summary was produced"+ } >> "$GITHUB_STEP_SUMMARY"++ - name: Collect observations and logs+ if: always()+ run: |+ mkdir -p _artifacts+ cp "$RUNNER_TEMP"/journey/*.json "$RUNNER_TEMP"/journey/*.log "$RUNNER_TEMP"/journey/*.jsonl _artifacts/ 2>/dev/null || true+ cp "$RUNNER_TEMP"/sam-one.log "$RUNNER_TEMP"/provider-*.log _artifacts/ 2>/dev/null || true++ - name: Upload observations+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: member-journey-${{ github.run_id }}+ path: _artifacts/+ retention-days: 14diff --git a/tests/integration/member_burst_test.go b/tests/integration/member_burst_test.gonew file mode 100644index 00000000..f3700a54--- /dev/null+++ b/tests/integration/member_burst_test.go@@ -0,0 +1,178 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package integration_test++import (+ "context"+ "fmt"+ "net"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strconv"+ "strings"+ "testing"+ "time"++ "github.com/google/sam/internal/bench"+ "github.com/google/sam/internal/standalone"+)++// TestMemberBurstJoinsConcurrently pins the join path under concurrency, the+// part of a new member's journey nothing else in the suite exercises: ten+// nodes enroll from one bootstrap token at the same instant, each+// authenticates to the router and calls a service another node advertises,+// and all ten complete. It is the integration tier of+// tests/scale/member-journey.sh, which takes the same journey at hundreds of+// members against a testnet; this one guards the enrollment race and the+// tool, not the latency.+func TestMemberBurstJoinsConcurrently(t *testing.T) {+ const members = 10+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer cancel()++ srv, err := standalone.New(standalone.Options{BindAddress: "127.0.0.1:0", DataDir: t.TempDir()})+ if err != nil {+ t.Fatalf("standalone.New: %v", err)+ }+ if err := srv.Start(ctx); err != nil {+ t.Fatalf("standalone start: %v", err)+ }+ t.Cleanup(func() { _ = srv.Close() })+ controlPlane := "http://" + srv.Addr()+ _, portStr, _ := net.SplitHostPort(srv.Addr())+ port, _ := strconv.Atoi(portStr)+ waitForActiveRouters(t, port, 1, 10*time.Second)+ joinToken := tokenPath(t, srv.JoinToken())++ // The service the members call: a real MCP server, which the provider's+ // advertisement probe needs before it publishes the record the members+ // will look up.+ backend := httptest.NewServer(newBoundaryMCPHandler(t))+ defer backend.Close()++ nodeBin := buildBinary(t, "./cmd/sam-node")+ providerDir := t.TempDir()+ provider := launchNode(t, nodeBin, os.Environ(), providerDir, "run",+ "--control-plane", controlPlane,+ "--data-dir", providerDir,+ "--bootstrap-token-path", joinToken,+ "--api-token-path", tokenPath(t, "test-token"),+ "--allow-loopback",+ "--config", writeNodeConfig(t, providerDir, nil, svcDecl{Type: "mcp", Name: "everything", TargetURL: backend.URL}),+ )+ provider.waitForAPI(t)++ // A short path: a member's socket lives under its directory, and a Unix+ // socket path has a hard limit.+ fleet, err := os.MkdirTemp("", "burst-")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = os.RemoveAll(fleet) })++ report, err := bench.RunJoin(ctx, bench.JoinOptions{+ NodeBin: nodeBin,+ Count: members,+ Dir: filepath.Join(fleet, "members"),+ ControlPlane: controlPlane,+ BootstrapTokenPath: joinToken,+ Service: "everything",+ ReadyTimeout: 30 * time.Second,+ CallTimeout: 30 * time.Second,+ MetricsBasePort: reservePortBlock(t, members),+ NodeArgs: []string{"--allow-loopback"},+ Log: testLogWriter{t},+ })+ if err != nil {+ t.Fatalf("RunJoin: %v", err)+ }++ if report.Completed != members || report.Failed != 0 {+ for _, m := range report.Members {+ if m.Error == "" {+ continue+ }+ t.Errorf("member %d: %s\n%s", m.Index, m.Error, logTail(filepath.Join(m.Dir, "node.log")))+ }+ t.Fatalf("completed %d of %d, failed %d: %v\nprovider log:\n%s", report.Completed, members, report.Failed, report.Errors, logTail(provider.logPath))+ }+ if report.ReadyMs.Count != members || report.JourneyMs.Count != members {+ t.Errorf("ready/journey samples = %d/%d, want %d each", report.ReadyMs.Count, report.JourneyMs.Count, members)+ }+ for _, m := range report.Members {+ if m.Provider != provider.peerID.String() {+ t.Errorf("member %d reached %q, want the provider %s", m.Index, m.Provider, provider.peerID)+ }+ }++ // Every member is a distinct enrollment the control plane holds: one+ // token, consumed once per member, with no row lost to the race.+ status := fetchAdminStatus(t, port, srv.AdminToken())+ enrolled := 0+ for _, n := range status.GetEnrolledNodes() {+ if n.GetPeerId() != provider.peerID.String() && !strings.EqualFold(n.GetPeerId(), srv.PeerID()) {+ enrolled+++ }+ }+ if enrolled < members {+ t.Errorf("control plane holds %d member enrollments, want %d", enrolled, members)+ }+ t.Logf("%d members: ready p50 %.0f ms p95 %.0f ms; journey p50 %.0f ms p95 %.0f ms max %.0f ms; %d call attempts",+ members, report.ReadyMs.P50, report.ReadyMs.P95, report.JourneyMs.P50, report.JourneyMs.P95, report.JourneyMs.Max, report.CallAttempts)+}++// reservePortBlock finds n consecutive free loopback ports in the reserved+// range and returns the first.+func reservePortBlock(t *testing.T, n int) int {+ t.Helper()+ for range 50 {+ base := getFreePort(t)+ free := true+ for i := 1; i < n; i++ {+ l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i))+ if err != nil {+ free = false+ break+ }+ _ = l.Close()+ }+ if free {+ return base+ }+ }+ t.Fatalf("no run of %d free ports", n)+ return 0+}++// logTail returns the last part of a log file, for a failure message.+func logTail(path string) string {+ data, err := os.ReadFile(path) // #nosec G304 -- a log the test wrote+ if err != nil {+ return err.Error()+ }+ if len(data) > 3000 {+ data = data[len(data)-3000:]+ }+ return string(data)+}++// testLogWriter routes a tool's progress lines into the test log.+type testLogWriter struct{ t *testing.T }++func (w testLogWriter) Write(p []byte) (int, error) {+ w.t.Log(strings.TrimRight(string(p), "\n"))+ return len(p), nil+}diff --git a/tests/scale/member-journey-summary.py b/tests/scale/member-journey-summary.pynew file mode 100644index 00000000..3a8e33e1--- /dev/null+++ b/tests/scale/member-journey-summary.py@@ -0,0 +1,178 @@+#!/usr/bin/env python3+#+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Render a member-journey result directory as one table and a verdict.++Usage: member-journey-summary.py DIR [JOURNEY_P50_MS JOURNEY_P95_MS OUTAGE_MAX_MS CP_P99_S]++Exit status 1 when any check fails, so the harness and CI can use it as a gate.+"""+import glob+import json+import os+import re+import sys++out = sys.argv[1]+t_p50, t_p95, t_outage, t_cp_p99 = (float(v) for v in (sys.argv[2:6] + ["5000", "15000", "60000", "1"][len(sys.argv) - 2:]))++verdicts = []+++def check(name, ok, detail):+ verdicts.append((name, ok, detail))+++def load(phase):+ path = os.path.join(out, phase + ".json")+ return json.load(open(path))["join"] if os.path.exists(path) else None+++print()+print("| phase | members | ready | completed | failed | ready p50/p95/max ms | journey p50/p95/max ms | calls/member |")+print("| --- | --- | --- | --- | --- | --- | --- | --- |")+left_behind = []+for phase in ("burst", "late", "late-after-rollout"):+ j = load(phase)+ if not j:+ continue+ r, jo = j["ready_ms"], j["journey_ms"]+ per = j["call_attempts"] / j["completed"] if j["completed"] else 0+ print(f"| {phase} | {j['count']} | {j['ready']} | {j['completed']} | {j['failed']} | "+ f"{r['p50']:.0f}/{r['p95']:.0f}/{r['max']:.0f} | {jo['p50']:.0f}/{jo['p95']:.0f}/{jo['max']:.0f} | {per:.2f} |")+ for stage, n in sorted(j.get("errors", {}).items()):+ left_behind.append(f"{phase}: {n} x {stage}")+ for msg, n in sorted(j.get("call_errors", {}).items()):+ left_behind.append(f"{phase}: {n} call attempt(s) failed with: {msg}")+ check(f"{phase}: every member completed", j["failed"] == 0, f"{j['failed']} of {j['count']} failed")+ if j["completed"]:+ check(f"{phase}: journey p50 <= {t_p50:.0f} ms", jo["p50"] <= t_p50, f"{jo['p50']:.0f} ms")+ check(f"{phase}: journey p95 <= {t_p95:.0f} ms", jo["p95"] <= t_p95, f"{jo['p95']:.0f} ms")+for line_ in left_behind:+ print(line_)++burst = load("burst")+h = burst.get("hold") if burst else None+if h:+ print()+ print(f"hold: {h['seconds']:.0f} s, {h['samples']} samples; resident {h['resident']}, fewest ready {h['min_ready']}; "+ f"{h['flaps']} flaps across {h['members_flapped']} members, outage p50/max "+ f"{h['outage_ms']['p50']:.0f}/{h['outage_ms']['max']:.0f} ms; "+ f"{h['members_not_ready_at_end']} not ready at the end, {h['members_exited']} exited")+ check("hold: every member ready at the end", h["members_not_ready_at_end"] == 0, f"{h['members_not_ready_at_end']} not ready")+ check("hold: no member process exited", h["members_exited"] == 0, f"{h['members_exited']} exited")+ check(f"hold: longest outage <= {t_outage/1000:.0f} s", h["outage_ms"]["max"] <= t_outage, f"{h['outage_ms']['max']/1000:.1f} s")++# What the mesh reported, as differences across the run.+line = re.compile(r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)')+++def parse(path):+ series = {}+ if not os.path.exists(path):+ return series+ for raw in open(path):+ m = line.match(raw)+ if not m:+ continue+ labels = tuple(sorted(re.findall(r'(\w+)="((?:[^"\\]|\\.)*)"', m.group(2) or "")))+ try:+ series[(m.group(1), labels)] = float(m.group(3))+ except ValueError:+ pass+ return series+++def parse_all(prefix, tag):+ # One exposition per pod; counters are summed across pods with the pod+ # name folded into the key, so a replica's own counter is only ever+ # differenced against itself.+ series = {}+ for path in sorted(glob.glob(os.path.join(out, f"{prefix}-*-{tag}.prom"))):+ pod = os.path.basename(path)[: -len(f"-{tag}.prom")]+ for (name, labels), v in parse(path).items():+ series[(name, labels + (("pod", pod),))] = v+ return series+++def total(series, name, **match):+ return sum(v for (n, labels), v in series.items()+ if n == name and all(dict(labels).get(k) == val for k, val in match.items()))+++def delta_by(before, after, name, label):+ keys = set()+ for (n, labels) in list(before) + list(after):+ if n == name:+ keys.add(dict(labels).get(label))+ return {k: round(total(after, name, **{label: k}) - total(before, name, **{label: k})) for k in sorted(keys, key=str)}+++def p99_of_delta(before, after, name):+ # Histogram buckets are cumulative counts; the difference across the run+ # is the run's own histogram, and p99 is the first bound holding 99%.+ buckets = {}+ for (n, labels), v in after.items():+ if n == name + "_bucket":+ le = dict(labels)["le"]+ buckets[le] = buckets.get(le, 0) + v - before.get((n, labels), 0)+ if not buckets or buckets.get("+Inf", 0) <= 0:+ return None+ target = 0.99 * buckets["+Inf"]+ for le in sorted(buckets, key=lambda s: float(s)):+ if buckets[le] >= target:+ return float(le)+ return float("inf")+++cp_before, cp_after = parse_all("sam-control-plane", "before"), parse_all("sam-control-plane", "after")+if cp_before and cp_after:+ print()+ codes = delta_by(cp_before, cp_after, "sam_control_plane_http_requests_total", "code")+ errors = sum(v for c, v in codes.items() if c and c.startswith("5"))+ p99 = p99_of_delta(cp_before, cp_after, "sam_control_plane_http_request_duration_seconds")+ # Both replicas export the same store-derived gauge; take one.+ nodes = tuple(max([v for (n, labels), v in s.items() if n == "sam_control_plane_enrolled_nodes" and dict(labels).get("role") == "sam:role:node"] or [0])+ for s in (cp_before, cp_after))+ print(f"control plane: requests by code {codes}; p99 <= {p99} s; enrolled nodes {nodes[0]:.0f} -> {nodes[1]:.0f}")+ check("control plane: no 5xx during the run", errors == 0, f"{errors:.0f} responses")+ if p99 is not None:+ check(f"control plane: request p99 <= {t_cp_p99:g} s", p99 <= t_cp_p99, f"<= {p99} s bucket")++routers = sorted({re.sub(r"-(before|after|resident|rolled)\.prom$", "", os.path.basename(p))+ for p in glob.glob(os.path.join(out, "sam-router-*.prom"))})+refused_total = 0+for r in routers:+ b, a = parse(os.path.join(out, f"{r}-before.prom")), parse(os.path.join(out, f"{r}-after.prom"))+ if not (b and a):+ continue+ refused = delta_by(b, a, "sam_router_inbound_connections_refused_total", "reason")+ handshakes = delta_by(b, a, "sam_router_auth_handshakes_total", "result")+ rss = (total(b, "process_resident_memory_bytes") / 2**20, total(a, "process_resident_memory_bytes") / 2**20)+ peers = total(a, "sam_router_authenticated_peers")+ refused_total += sum(refused.values())+ print(f"{r}: refused {refused}; handshakes {handshakes}; rss {rss[0]:.0f} -> {rss[1]:.0f} MiB; authenticated peers now {peers:.0f}")+if routers:+ check("routers: no inbound connection refused", refused_total == 0,+ f"{refused_total:.0f} refused (a per-source-IP limit shows here first)")++print()+failed = 0+for name, ok, detail in verdicts:+ print(f"{'PASS' if ok else 'FAIL'} {name} ({detail})")+ failed += 0 if ok else 1+print()+print(f"{len(verdicts) - failed} of {len(verdicts)} checks passed; observations under {out}")+sys.exit(1 if failed else 0)diff --git a/tests/scale/member-journey.sh b/tests/scale/member-journey.shnew file mode 100755index 00000000..33baf86e--- /dev/null+++ b/tests/scale/member-journey.sh@@ -0,0 +1,339 @@+#!/usr/bin/env bash+#+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+# What does the Nth member experience?+#+# The density and fleet runs say how many agents one member can carry. This+# run is about members: N sam-node processes join a real mesh from one host,+# each is timed from start to its first call of a service through the mesh,+# the fleet then stays resident while more members join and, if asked, while+# the routers are restarted under it. Every member on this host shares one+# source address, which is what a classroom or an office looks like to the+# router.+#+# Phases, each written as its own observation under --out:+# burst N members start at --ramp per second burst.json+# late --late more join the populated mesh late.json+# rollout (--rollout) restart the router StatefulSet, events.jsonl+# then --late more join behind it late-after-rollout.json+# hold the burst fleet stays for --hold, sampled burst.json (hold section)+#+# Then one table and a verdict against the thresholds below, which the+# environment overrides (T_JOURNEY_P95_MS=20000 ...).+#+# Usage:+# tests/scale/member-journey.sh --env bananas --count 500 --late 20 --hold 1h --rollout --out /var/log/journey+# tests/scale/member-journey.sh --control-plane http://127.0.0.1:8080 --bootstrap-token-path ./join-token \+# --node-arg=--insecure-control-plane --node-arg=--allow-loopback --count 20 --out /tmp/journey+#+# The members call an MCP service named --service, so some node on the mesh+# has to advertise one; on a testnet that is the everything canary. For a+# local mesh, tests/scale/stdio-mcp.py (Python SDK) and tests/scale/stdio-mcp+# (Go SDK) are real servers a provider node can run as a command backend;+# .github/workflows/member-journey.yaml shows the whole local setup.+#+# --env mints the bootstrap token from the control plane's admin secret and+# snapshots the control plane's and routers' metrics around the run through+# the API server. Without it, bring your own token; the verdict then covers+# only what the members themselves saw.++set -euo pipefail++COUNT=50+LATE=10+RAMP=0+HOLD=10m+SERVICE=everything+ENV_NAME=""+CONTROL_PLANE=""+BOOTSTRAP_TOKEN_PATH=""+OUT=""+ROLLOUT=0+NODE_ARGS=()++REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+SAM_NODE="${SAM_NODE:-${REPO_ROOT}/bin/sam-node}"+SAM_BENCH="${SAM_BENCH:-${REPO_ROOT}/bin/sam-bench}"+# May carry arguments, e.g. KUBECTL="kubectl --context my-cluster".+KUBECTL="${KUBECTL:-kubectl}"++# Thresholds. The journey bound is the SDKs' dial timeout: a member that+# takes longer than that would have failed as an SDK agent.+T_JOURNEY_P50_MS="${T_JOURNEY_P50_MS:-5000}"+T_JOURNEY_P95_MS="${T_JOURNEY_P95_MS:-15000}"+T_OUTAGE_MAX_MS="${T_OUTAGE_MAX_MS:-60000}"+T_CP_P99_S="${T_CP_P99_S:-1}"++usage() {+ sed -n '17,46p' "$0" | sed 's/^# \{0,1\}//'+ cat <<EOF++Options:+ --env NAME testnet (hub, bananas): mints the token, snapshots metrics, enables --rollout+ --control-plane URL control plane to enroll with (default https://NAME.sam-mesh.dev with --env)+ --bootstrap-token-path F bootstrap token to enroll with; minted from the admin secret with --env if omitted+ --count N members in the burst (default ${COUNT})+ --late N members that join once the burst is resident, per late phase (default ${LATE}; 0 skips)+ --ramp N members started per second (default ${RAMP}: all at once)+ --hold DURATION how long the burst fleet stays resident (default ${HOLD})+ --rollout restart the router StatefulSet while the fleet is resident (needs --env)+ --service NAME MCP service each member finds and calls (default ${SERVICE})+ --node-arg ARG extra sam-node argument, repeatable+ --out DIR where observations, logs and member state land (required)+EOF+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --env) ENV_NAME="$2"; shift 2 ;;+ --control-plane) CONTROL_PLANE="$2"; shift 2 ;;+ --bootstrap-token-path) BOOTSTRAP_TOKEN_PATH="$2"; shift 2 ;;+ --count) COUNT="$2"; shift 2 ;;+ --late) LATE="$2"; shift 2 ;;+ --ramp) RAMP="$2"; shift 2 ;;+ --hold) HOLD="$2"; shift 2 ;;+ --rollout) ROLLOUT=1; shift ;;+ --service) SERVICE="$2"; shift 2 ;;+ --node-arg) NODE_ARGS+=("$2"); shift 2 ;;+ --node-arg=*) NODE_ARGS+=("${1#--node-arg=}"); shift ;;+ --out) OUT="$2"; shift 2 ;;+ -h|--help) usage; exit 0 ;;+ *) echo "unknown flag: $1" >&2; usage >&2; exit 2 ;;+ esac+done++fail() { echo "member-journey: $*" >&2; exit 1; }+log() { echo "$(date -u +%FT%TZ) $*"; }++[[ -n "$OUT" ]] || fail "--out is required"+[[ -x "$SAM_NODE" ]] || fail "sam-node not found at $SAM_NODE; run make or set SAM_NODE"+[[ -x "$SAM_BENCH" ]] || fail "sam-bench not found at $SAM_BENCH; run make or set SAM_BENCH"+command -v python3 >/dev/null || fail "python3 is needed for the summary"+if [[ -n "$ENV_NAME" ]]; then+ NS="sam-${ENV_NAME}"+ command -v "${KUBECTL%% *}" >/dev/null || fail "kubectl is needed with --env"+ $KUBECTL get namespace "$NS" >/dev/null 2>&1 || fail "namespace $NS is not reachable with the current kubectl context"+ CONTROL_PLANE="${CONTROL_PLANE:-https://${ENV_NAME}.sam-mesh.dev}"+else+ [[ -n "$CONTROL_PLANE" ]] || fail "--control-plane is required without --env"+ [[ -n "$BOOTSTRAP_TOKEN_PATH" ]] || fail "--bootstrap-token-path is required without --env"+ [[ "$ROLLOUT" -eq 0 ]] || fail "--rollout needs --env"+fi++mkdir -p "$OUT"+for phase in burst late late-after-rollout; do+ [[ -e "$OUT/$phase" ]] && fail "$OUT/$phase exists; a member directory with an identity would resume rather than join. Use a fresh --out"+done++# Every member is a process with its own sockets and a libp2p host. The soft+# limit is per process, so this only has to cover one node, but it is often 1024.+ulimit -n 65536 2>/dev/null || log "could not raise the file descriptor limit; a member may run out"++JOIN_PID=""+PF_PID=""+TOKEN_ID=""+SUMMARIZED=0+cleanup() {+ local status=$?+ if [[ -n "$JOIN_PID" ]] && kill -0 "$JOIN_PID" 2>/dev/null; then+ # SIGINT ends the hold; the run still writes its report and stops its members.+ kill -INT "$JOIN_PID" 2>/dev/null || true+ wait "$JOIN_PID" 2>/dev/null || true+ fi+ JOIN_PID=""+ [[ -n "$PF_PID" ]] && kill "$PF_PID" 2>/dev/null || true+ PF_PID=""+ if [[ "$SUMMARIZED" -eq 0 && -f "$OUT/burst.json" ]]; then+ summarize || true+ fi+ [[ -n "$TOKEN_ID" ]] && revoke_token+ rm -f "$OUT/admin-header"+ exit "$status"+}+trap cleanup EXIT++# admin_api runs curl against the control plane's admin API over a+# port-forward, with the admin token in a header file rather than on a+# command line. Arguments are curl's, after the method and path.+admin_api() {+ local method="$1" path="$2"+ shift 2+ local port+ port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])')+ if [[ ! -s "$OUT/admin-header" ]]; then+ $KUBECTL get secret "sam-control-plane-secret-${ENV_NAME}" -n "$NS" -o jsonpath='{.data.admin-token}' \+ | base64 -d | sed 's/^/Authorization: Bearer /' > "$OUT/admin-header"+ chmod 600 "$OUT/admin-header"+ fi+ $KUBECTL port-forward "deployment/sam-control-plane-${ENV_NAME}" -n "$NS" "${port}:8080" >/dev/null 2>&1 &+ PF_PID=$!+ for _ in $(seq 1 30); do+ curl -fsS -o /dev/null "http://127.0.0.1:${port}/healthz" 2>/dev/null && break+ sleep 0.5+ done+ local status=0+ curl -fsS -X "$method" -H "@${OUT}/admin-header" "$@" "http://127.0.0.1:${port}${path}" || status=$?+ kill "$PF_PID" 2>/dev/null || true+ PF_PID=""+ return "$status"+}++# mint_token writes a bootstrap token with enough usages for every phase and+# remembers its id so the run can revoke it on the way out.+mint_token() {+ local uses=$(( COUNT + 2 * LATE + 5 ))+ admin_api POST /admin/bootstrap-tokens -H 'Content-Type: application/json' \+ -d "{\"role\":\"sam:role:node\",\"max_usages\":${uses},\"ttl_hours\":48,\"description\":\"member-journey ${COUNT}+${LATE} from $(hostname)\"}" \+ > "$OUT/token.json" || fail "minting a bootstrap token failed"+ python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["token"])' "$OUT/token.json" > "$OUT/bootstrap-token"+ TOKEN_ID=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["id"])' "$OUT/token.json")+ rm -f "$OUT/token.json"+ chmod 600 "$OUT/bootstrap-token"+ [[ -s "$OUT/bootstrap-token" && -n "$TOKEN_ID" ]] || fail "no bootstrap token came back"+ BOOTSTRAP_TOKEN_PATH="$OUT/bootstrap-token"+ log "minted bootstrap token ${TOKEN_ID} for ${uses} enrollments"+}++# revoke_token retires the token a run minted: a test must not leave a+# standing credential on a public testnet.+revoke_token() {+ if admin_api DELETE "/admin/bootstrap-tokens/${TOKEN_ID}" -o /dev/null; then+ log "revoked bootstrap token ${TOKEN_ID}"+ else+ log "could not revoke bootstrap token ${TOKEN_ID}; revoke it by hand"+ fi+ TOKEN_ID=""+ rm -f "$OUT/bootstrap-token"+}++# snapshot reads what every control plane and router pod exports, through+# the API server so no port is opened for it. Per pod, not per Service: the+# control plane has two replicas and a Service proxy alternates between+# them, which would make a before/after difference meaningless. The+# members' own numbers say what they experienced; these say what the mesh+# did to produce it.+snapshot() {+ local tag="$1" pod name+ [[ -n "$ENV_NAME" ]] || return 0+ for pod in $($KUBECTL get pods -n "$NS" -l "app=sam-control-plane-${ENV_NAME}" -o name 2>/dev/null); do+ name="${pod#pod/}"+ $KUBECTL get --raw "/api/v1/namespaces/${NS}/pods/${name}:8080/proxy/metrics" \+ > "$OUT/${name}-${tag}.prom" 2>/dev/null || log "could not read ${name} metrics (${tag})"+ done+ for pod in $($KUBECTL get pods -n "$NS" -l "app=sam-router-${ENV_NAME}" -o name 2>/dev/null); do+ name="${pod#pod/}"+ $KUBECTL get --raw "/api/v1/namespaces/${NS}/pods/${name}:9090/proxy/metrics" \+ > "$OUT/${name}-${tag}.prom" 2>/dev/null || log "could not read ${name} metrics (${tag})"+ done+}++event() {+ printf '{"event":"%s","at":"%s"}\n' "$1" "$(date -u +%FT%TZ)" >> "$OUT/events.jsonl"+}++# summarize closes the run: a last snapshot, then the table and verdict. It+# runs from the exit trap too, so an interrupted hold still gets its report.+summarize() {+ SUMMARIZED=1+ event hold-end+ snapshot after+ python3 "${REPO_ROOT}/tests/scale/member-journey-summary.py" "$OUT" \+ "$T_JOURNEY_P50_MS" "$T_JOURNEY_P95_MS" "$T_OUTAGE_MAX_MS" "$T_CP_P99_S"+}++# join_cmd fills JOIN_CMD with one phase's sam-bench invocation. It is an+# array rather than a function so the burst can be backgrounded directly:+# a backgrounded function is a subshell, and a signal to it would not reach+# the run inside.+JOIN_CMD=()+join_cmd() { # phase count base-port extra...+ local phase="$1" count="$2" base="$3" a+ shift 3+ JOIN_CMD=("$SAM_BENCH" join+ --node-bin "$SAM_NODE"+ --control-plane "$CONTROL_PLANE"+ --bootstrap-token-path "$BOOTSTRAP_TOKEN_PATH"+ --service "$SERVICE"+ --count "$count"+ --metrics-base-port "$base"+ --dir "$OUT/$phase"+ --label "phase=$phase" --label "count=$count" --label "burst=$COUNT" --label "host=$(hostname)"+ --out "$OUT/$phase.json")+ for a in "${NODE_ARGS[@]}"; do JOIN_CMD+=("--node-arg=$a"); done+ JOIN_CMD+=("$@")+}++[[ -n "$BOOTSTRAP_TOKEN_PATH" ]] || mint_token+rm -f "$OUT/resident"+: > "$OUT/events.jsonl"+{+ echo "control_plane=$CONTROL_PLANE"+ echo "host=$(hostname) nproc=$(nproc) mem_kib=$(awk '/MemTotal/{print $2}' /proc/meminfo)"+ echo "sam_node=$("$SAM_NODE" --version 2>/dev/null | head -1)"+} > "$OUT/environment.txt"++snapshot before+event burst-start+log "burst: ${COUNT} members at ${RAMP}/s, then holding ${HOLD}"+join_cmd burst "$COUNT" 20000 --ramp "$RAMP" --hold "$HOLD" --resident-marker "$OUT/resident"+"${JOIN_CMD[@]}" 2> "$OUT/burst.log" &+JOIN_PID=$!+while [[ ! -f "$OUT/resident" ]]; do+ kill -0 "$JOIN_PID" 2>/dev/null || fail "the burst ended before the fleet was resident; see $OUT/burst.log"+ sleep 2+done+event burst-resident+snapshot resident+python3 - "$OUT/burst.json" <<'PY'+import json, sys+j = json.load(open(sys.argv[1]))["join"]+print(f"burst: {j['ready']}/{j['count']} ready, {j['completed']} completed, {j['failed']} failed; "+ f"journey p50 {j['journey_ms']['p50']:.0f} ms, p95 {j['journey_ms']['p95']:.0f} ms, max {j['journey_ms']['max']:.0f} ms")+for stage, n in sorted(j.get("errors", {}).items()):+ print(f" {n} x {stage}")+PY++if (( LATE > 0 )); then+ event late-start+ log "late: ${LATE} members join the populated mesh"+ join_cmd late "$LATE" 30000+ "${JOIN_CMD[@]}" 2> "$OUT/late.log" || log "late phase did not run cleanly; see $OUT/late.log"+ event late-end+fi++if (( ROLLOUT )); then+ event rollout-start+ log "rollout: restarting statefulset/sam-router-${ENV_NAME} under the resident fleet"+ $KUBECTL rollout restart "statefulset/sam-router-${ENV_NAME}" -n "$NS"+ $KUBECTL rollout status "statefulset/sam-router-${ENV_NAME}" -n "$NS" --timeout=600s || log "rollout did not finish in 600s"+ event rollout-end+ snapshot rolled+ if (( LATE > 0 )); then+ # The routers' addresses just changed and the DNS records catch up on a+ # timer; a long bound here measures that wait instead of hiding it.+ event late-after-rollout-start+ log "late: ${LATE} members join behind the rollout"+ join_cmd late-after-rollout "$LATE" 31000 --ready-timeout 600s+ "${JOIN_CMD[@]}" 2> "$OUT/late-after-rollout.log" \+ || log "late-after-rollout phase did not run cleanly; see $OUT/late-after-rollout.log"+ event late-after-rollout-end+ fi+fi++log "holding; the burst fleet stays until ${HOLD} after it became resident (Ctrl-C ends it early with a report)"+wait "$JOIN_PID" || log "burst run exited non-zero; see $OUT/burst.log"+JOIN_PID=""+summarizediff --git a/tests/scale/stdio-mcp.py b/tests/scale/stdio-mcp.pynew file mode 100644index 00000000..39f4f00e--- /dev/null+++ b/tests/scale/stdio-mcp.py@@ -0,0 +1,42 @@+#!/usr/bin/env python3+#+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""An MCP server over stdio, for a provider node in a local mesh.++Built on the official Python SDK (the `mcp` package, as in+development/examples), so what a member reaches through the mesh is a real+server and not an imitation of one. A node declares it as a command backend+and advertises it; the member-journey harness calls it as "everything".++ pip install "mcp>=2,<3"++ services:+ - type: mcp+ name: everything+ command: ["python3", "tests/scale/stdio-mcp.py"]+"""+from mcp.server.mcpserver import MCPServer++mcp = MCPServer("everything")+++def echo(text: str) -> str:+ """Return the text given."""+ return text+++if __name__ == "__main__":+ mcp.run(transport="stdio")diff --git a/tests/scale/stdio-mcp/main.go b/tests/scale/stdio-mcp/main.gonew file mode 100644index 00000000..75d0c093--- /dev/null+++ b/tests/scale/stdio-mcp/main.go@@ -0,0 +1,50 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++// Command stdio-mcp is an MCP server over stdio, for a provider node in a+// local mesh. It is the Go twin of tests/scale/stdio-mcp.py, built on the+// official Go SDK, so a member's first call through the mesh lands on a real+// server in either language.+//+// go build -o bin/stdio-mcp ./tests/scale/stdio-mcp+//+// services:+// - type: mcp+// name: everything+// command: ["bin/stdio-mcp"]+package main++import (+ "context"+ "fmt"+ "os"++ "github.com/modelcontextprotocol/go-sdk/mcp"+)++type echoInput struct {+ Text string `json:"text" jsonschema:"the text to return"`+}++func main() {+ server := mcp.NewServer(&mcp.Implementation{Name: "everything", Version: "0"}, nil)+ mcp.AddTool(server, &mcp.Tool{Name: "echo", Description: "Return the text given"},+ func(_ context.Context, _ *mcp.CallToolRequest, in echoInput) (*mcp.CallToolResult, any, error) {+ return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: in.Text}}}, nil, nil+ })+ if err := server.Run(context.Background(), &mcp.StdioTransport{}); err != nil {+ fmt.Fprintln(os.Stderr, "stdio-mcp:", err)+ os.Exit(1)+ }+}