Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
bench: add sam-bench join, the journey of the Nth member
Antonio Ojeacd5d80ad5ca1
diff --git a/cmd/sam-bench/main.go b/cmd/sam-bench/main.goindex a04d4bf3..242b6ad9 100644--- a/cmd/sam-bench/main.go+++ b/cmd/sam-bench/main.go@@ -150,7 +150,7 @@ func main() {if err := runCmd.MarkFlagRequired("target"); err != nil {panic(err)}- rootCmd.AddCommand(runCmd, newReportCmd(), newSTSCmd())+ rootCmd.AddCommand(runCmd, newReportCmd(), newSTSCmd(), newJoinCmd())ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)defer stop()@@ -197,6 +197,102 @@ func newSTSCmd() *cobra.Command {return cmd}+// joinObservation is one fleet's journey, with what the mesh's own processes+// reported before and after it, in the same shape as an observation.+type joinObservation struct {+ Labels map[string]string `json:"labels,omitempty"`+ Started time.Time `json:"started"`+ Join *bench.JoinReport `json:"join"`+ Before map[string]map[string]float64 `json:"metrics_before,omitempty"`+ After map[string]map[string]float64 `json:"metrics_after,omitempty"`+}++func newJoinCmd() *cobra.Command {+ var (+ opts bench.JoinOptions+ marker string+ scrape []string+ labels []string+ out string+ )+ cmd := &cobra.Command{+ Use: "join",+ Short: "Start a fleet of members and record what joining cost each of them",+ Long: "Starts N sam-node processes from one bootstrap token and times each one from start\n" ++ "to an authenticated router connection to its first call of a service through the\n" ++ "mesh. With --hold the fleet then stays resident and every member's readiness is\n" ++ "sampled, so a router rollout or a key rotation that happens meanwhile shows up as\n" ++ "outage windows. The processes all share this host's source address.",+ SilenceUsage: true,+ RunE: func(cmd *cobra.Command, _ []string) error {+ tags, err := parseLabels(labels)+ if err != nil {+ return err+ }+ if opts.Dir == "" {+ if opts.Dir, err = os.MkdirTemp("", "sam-join-"); err != nil {+ return err+ }+ }+ opts.Log = os.Stderr++ obs := joinObservation{Labels: tags, Started: time.Now()}+ // A hold can run for a day; the join numbers are written as soon+ // as they exist and the marker tells a script the fleet is up.+ opts.OnResident = func(r *bench.JoinReport) {+ obs.Join = r+ if out != "" {+ if err := write(out, obs); err != nil {+ fmt.Fprintf(os.Stderr, "sam-bench: %v\n", err)+ }+ }+ if marker != "" {+ stamp := []byte(time.Now().UTC().Format(time.RFC3339) + "\n")+ if err := os.WriteFile(marker, stamp, 0o600); err != nil {+ fmt.Fprintf(os.Stderr, "sam-bench: %v\n", err)+ }+ }+ }+ obs.Before, err = scrapeAll(cmd.Context(), scrape)+ if err != nil {+ return err+ }+ obs.Join, err = bench.RunJoin(cmd.Context(), opts)+ if err != nil {+ return err+ }+ // The fleet is gone by now; a scrape that fails here would lose the+ // report over a metrics endpoint, so it is reported and not fatal.+ if obs.After, err = scrapeAll(context.Background(), scrape); err != nil {+ fmt.Fprintf(os.Stderr, "sam-bench: metrics after the run: %v\n", err)+ }+ return write(out, obs)+ },+ }+ flags := cmd.Flags()+ flags.StringVar(&opts.NodeBin, "node-bin", "sam-node", "sam-node binary to start one process of per member")+ flags.IntVar(&opts.Count, "count", 10, "Members to start")+ flags.StringVar(&opts.Dir, "dir", "", "Empty directory for member state, sockets and logs; default a new temporary one")+ flags.StringVar(&opts.ControlPlane, "control-plane", "", "Control plane URL every member enrolls with (required)")+ flags.StringVar(&opts.BootstrapTokenPath, "bootstrap-token-path", "", "File holding the bootstrap token the members enroll with; it needs --count usages left")+ flags.StringVar(&opts.Service, "service", "everything", "MCP service each member finds and calls once ready; empty ends the journey at readiness")+ flags.Float64Var(&opts.Ramp, "ramp", 0, "Members started per second; 0 starts them all at once")+ flags.DurationVar(&opts.ReadyTimeout, "ready-timeout", 180*time.Second, "Bound on one member's enrollment and router authentication")+ flags.DurationVar(&opts.CallTimeout, "call-timeout", 120*time.Second, "Bound on one member's discovery and first call, from readiness")+ flags.DurationVar(&opts.Hold, "hold", 0, "Keep the fleet resident this long after the journeys end, sampling readiness; 0 tears it down at once")+ flags.DurationVar(&opts.SampleInterval, "sample-interval", 10*time.Second, "How often each resident member's readiness is read during --hold")+ flags.IntVar(&opts.MetricsBasePort, "metrics-base-port", 20000, "Member i serves /readyz on 127.0.0.1:(port+i)")+ flags.StringArrayVar(&opts.NodeArgs, "node-arg", nil, "Extra argument for every sam-node, repeatable (e.g. --node-arg=--insecure-control-plane)")+ flags.StringVar(&marker, "resident-marker", "", "File to create once the fleet is resident and the hold has begun")+ flags.StringArrayVar(&scrape, "scrape", nil, "Metrics endpoint to record before and after the run, repeatable")+ flags.StringArrayVar(&labels, "label", nil, "Condition to record with the observation as name=value, repeatable")+ flags.StringVar(&out, "out", "", "File to write the observation to; default stdout")+ if err := cmd.MarkFlagRequired("control-plane"); err != nil {+ panic(err)+ }+ return cmd+}+// scrapeAll records every endpoint, refusing to continue if one is missing:// an observation with a hole in it is worse than no observation, because it// still looks like data.@@ -246,7 +342,7 @@ func parseLabels(raw []string) (map[string]string, error) {return labels, nil}-func write(path string, obs observation) error {+func write(path string, obs any) error {encoded, err := json.MarshalIndent(obs, "", " ")if err != nil {return errdiff --git a/internal/bench/join.go b/internal/bench/join.gonew file mode 100644index 00000000..32e61164--- /dev/null+++ b/internal/bench/join.go@@ -0,0 +1,807 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package bench++import (+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net"+ "net/http"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "sync"+ "syscall"+ "time"++ "github.com/google/sam/api"+)++// A density run says how many agents one member can carry. This asks the+// other question: what the Nth member experiences while N-1 are already+// there. The journey measured is the one a new user takes: start a node with+// nothing but a bootstrap token, wait until it holds an authenticated router+// connection, find a service someone else advertised and make one call to it+// through the mesh. Each step is timed per member, so a burst of joiners that+// leaves a few behind is reported as a few left behind rather than as a+// slightly worse mean.+//+// The members are real sam-node processes, one per member, started from this+// process so that every timestamp comes from one clock. They all share this+// host's source address, which is what a classroom, an office or a CI runner+// pool looks like from the router.++// JoinOptions describe one fleet to take through the join journey.+type JoinOptions struct {+ // NodeBin is the sam-node binary; one process of it is started per member.+ NodeBin string++ // Count is how many members to start.+ Count int++ // Dir is the directory under which each member gets its own state,+ // socket and log. It must be empty: a member directory that already holds+ // an identity would resume rather than join.+ Dir string++ // ControlPlane is the URL every member enrolls with.+ ControlPlane string++ // BootstrapTokenPath is the file holding the bootstrap token the members+ // enroll with. It needs at least Count usages left. Empty means the+ // credential comes from NodeArgs instead, such as a --jwt-path.+ BootstrapTokenPath string++ // Service is the MCP service each member looks up and calls once it is+ // ready. Empty ends the journey at readiness.+ Service string++ // Ramp is how many members to start per second. Zero starts them all at+ // once.+ Ramp float64++ // ReadyTimeout bounds enrollment plus router authentication for one+ // member.+ ReadyTimeout time.Duration++ // CallTimeout bounds discovery plus the first successful call for one+ // member, counted from when it became ready.+ CallTimeout time.Duration++ // Hold keeps the fleet resident after the last journey has ended and+ // samples every member's readiness until it elapses or the context is+ // cancelled. Zero tears the fleet down as soon as the journeys end.+ Hold time.Duration++ // SampleInterval is how often each resident member is read during Hold.+ SampleInterval time.Duration++ // MetricsBasePort is the first of Count consecutive loopback ports;+ // member i serves /readyz on MetricsBasePort+i.+ MetricsBasePort int++ // NodeArgs are appended to every member's command line as given.+ NodeArgs []string++ // OnResident, if set, is called with the report once the journeys have+ // ended and the hold has begun. The report is complete except for Hold,+ // so a caller can publish the join numbers before a long hold ends.+ OnResident func(*JoinReport)++ // Log receives one line per event: a member's journey ending, a resident+ // member losing or regaining its router. Nil discards them.+ Log io.Writer+}++// JoinReport is the outcome of one fleet's journey.+type JoinReport struct {+ Count int `json:"count"`+ Ramp float64 `json:"ramp_per_second"`+ Service string `json:"service,omitempty"`++ // Ready counts members that reached /readyz 200. Completed counts those+ // that finished the whole journey, which is readiness alone when there+ // is no Service. Failed is the rest.+ Ready int `json:"ready"`+ Completed int `json:"completed"`+ Failed int `json:"failed"`++ // Elapsed runs from the first member starting to the last journey ending.+ Elapsed float64 `json:"elapsed_seconds"`++ // ReadyMs is process start to /readyz 200, FirstCallMs is readiness to+ // the first successful call, and JourneyMs is their sum, each over the+ // members that got that far.+ ReadyMs Distribution `json:"ready_ms"`+ FirstCallMs Distribution `json:"first_call_ms"`+ JourneyMs Distribution `json:"journey_ms"`++ // CallAttempts is the number of provider calls made across the fleet.+ // More than one per completed member means provider records that named+ // a peer which did not answer; CallErrors says how each of those+ // attempts failed, including on members that then succeeded elsewhere.+ CallAttempts int `json:"call_attempts"`+ CallErrors map[string]int `json:"call_errors,omitempty"`++ // Errors counts failures by stage, so a fleet that mostly failed cannot+ // be mistaken for a fast one.+ Errors map[string]int `json:"errors,omitempty"`++ Hold *HoldReport `json:"hold,omitempty"`++ Members []MemberResult `json:"members"`+}++// MemberResult is one member's journey.+type MemberResult struct {+ Index int `json:"index"`+ Dir string `json:"dir"`+ ReadyMs float64 `json:"ready_ms,omitempty"`+ FirstCallMs float64 `json:"first_call_ms,omitempty"`+ Provider string `json:"provider,omitempty"`+ Attempts int `json:"call_attempts,omitempty"`+ Error string `json:"error,omitempty"`+}++// HoldReport describes a resident fleet over time.+type HoldReport struct {+ Seconds float64 `json:"seconds"`+ Samples int `json:"samples"`++ // Resident is how many members were ready at the first sample and+ // MinReady the fewest at any sample.+ Resident int `json:"resident"`+ MinReady int `json:"min_ready"`++ // Flaps counts ready-to-not-ready transitions across the fleet, Flapped+ // the members that had at least one, NotReadyAtEnd those still down when+ // the hold ended and Exited the processes that ended by themselves.+ Flaps int `json:"flaps"`+ Flapped int `json:"members_flapped"`+ NotReadyAtEnd int `json:"members_not_ready_at_end"`+ Exited int `json:"members_exited"`++ // OutageMs is the length of every not-ready window that ended during+ // the hold. A window still open at the end is in NotReadyAtEnd instead.+ OutageMs Distribution `json:"outage_ms"`++ // Series is the ready count at each sample, for lining up against the+ // events an operator caused while the fleet was resident.+ Series []ReadySample `json:"ready_series"`+}++// ReadySample is the fleet's ready count at one instant.+type ReadySample struct {+ At time.Time `json:"at"`+ Ready int `json:"ready"`+}++// Journey stages, used as the keys of JoinReport.Errors.+const (+ stageStart = "start failed"+ stageReady = "not ready in time"+ stageExited = "process exited"+ stageCall = "no provider answered in time"+ stageCanceled = "cancelled"+)++const (+ // Readiness is polled from readyPollMin, backing off to readyPollMax: a+ // member that is up in a few hundred milliseconds is measured as such,+ // and one that takes a minute is not asked a thousand times.+ readyPollMin = 50 * time.Millisecond+ readyPollMax = 500 * time.Millisecond+ callRetryInterval = 5 * time.Second+ requestTimeout = 30 * time.Second+ teardownGrace = 10 * time.Second+ sampleConcurrency = 64+ // maxBodyProbe bounds how much of a response is read looking for the+ // MCP server's name; an initialize answer is a few hundred bytes.+ maxBodyProbe = 64 << 10+)++// mcpInitialize is the first request of an MCP session, enough for the+// server to name itself.+const mcpInitialize = `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"sam-bench","version":"0"}}}`++type member struct {+ index int+ dir string+ socket string+ metricsAddr string+ readyz string+ logPath string+ cmd *exec.Cmd+ logFile *os.File+ started time.Time+ exited chan struct{}+ exitErr error++ readyAt time.Time+ calledAt time.Time+ provider string+ attempts int+ callErrors map[string]int+ stage string+ detail string+}++func (m *member) fail(stage, detail string) {+ m.stage, m.detail = stage, detail+}++func (m *member) hasExited() bool {+ if m.exited == nil {+ return true+ }+ select {+ case <-m.exited:+ return true+ default:+ return false+ }+}++func (m *member) result() MemberResult {+ r := MemberResult{Index: m.index, Dir: m.dir, Provider: m.provider, Attempts: m.attempts}+ if !m.readyAt.IsZero() {+ r.ReadyMs = millis(m.readyAt.Sub(m.started))+ }+ if !m.calledAt.IsZero() {+ r.FirstCallMs = millis(m.calledAt.Sub(m.readyAt))+ }+ if m.stage != "" {+ r.Error = m.stage + ": " + m.detail+ }+ return r+}++// RunJoin starts the fleet, takes every member through the journey, holds+// the fleet if asked, tears it down and reports. Cancelling the context ends+// whatever phase is running; the report still describes what happened.+func RunJoin(ctx context.Context, opts JoinOptions) (*JoinReport, error) {+ if err := opts.validate(); err != nil {+ return nil, err+ }+ opts.setDefaults()+ if err := os.MkdirAll(opts.Dir, 0o700); err != nil {+ return nil, fmt.Errorf("bench: %w", err)+ }++ members := make([]*member, 0, opts.Count)+ defer func() { teardown(members) }()++ first := time.Now()+ var wg sync.WaitGroup+ for i := range opts.Count {+ if i > 0 && opts.Ramp > 0 {+ select {+ case <-time.After(time.Duration(float64(time.Second) / opts.Ramp)):+ case <-ctx.Done():+ }+ }+ m := newMember(opts, i)+ members = append(members, m)+ if ctx.Err() != nil {+ m.fail(stageCanceled, "before start")+ continue+ }+ if err := m.start(opts); err != nil {+ m.fail(stageStart, err.Error())+ continue+ }+ wg.Add(1)+ go func() {+ defer wg.Done()+ journey(ctx, opts, m)+ }()+ }+ wg.Wait()++ report := summariseJoin(opts, members, time.Since(first))+ if opts.Hold > 0 && ctx.Err() == nil {+ if opts.OnResident != nil {+ opts.OnResident(report)+ }+ opts.logf("fleet resident: %d of %d ready; holding for %s", report.Ready, report.Count, opts.Hold)+ report.Hold = hold(ctx, opts, members)+ }+ return report, nil+}++func (o *JoinOptions) validate() error {+ switch {+ case o.NodeBin == "":+ return errors.New("bench: no sam-node binary")+ case o.Count <= 0:+ return errors.New("bench: no members to start")+ case o.Dir == "":+ return errors.New("bench: no directory for member state")+ case o.ControlPlane == "":+ return errors.New("bench: no control plane")+ case o.Ramp < 0:+ return errors.New("bench: ramp must not be negative")+ }+ entries, err := os.ReadDir(o.Dir)+ if err != nil && !errors.Is(err, os.ErrNotExist) {+ return fmt.Errorf("bench: %w", err)+ }+ if len(entries) > 0 {+ return fmt.Errorf("bench: %s is not empty; a member directory that already holds an identity would resume rather than join", o.Dir)+ }+ // A Unix socket path is limited to 108 bytes on Linux; the member's own+ // directory and socket name take some of that.+ if len(filepath.Join(o.Dir, "member-0000", "node.sock")) > 100 {+ return fmt.Errorf("bench: %s is too deep for a member's socket path", o.Dir)+ }+ return nil+}++func (o *JoinOptions) setDefaults() {+ if o.ReadyTimeout <= 0 {+ o.ReadyTimeout = 180 * time.Second+ }+ if o.CallTimeout <= 0 {+ o.CallTimeout = 120 * time.Second+ }+ if o.SampleInterval <= 0 {+ o.SampleInterval = 10 * time.Second+ }+ if o.MetricsBasePort <= 0 {+ o.MetricsBasePort = 20000+ }+}++func (o *JoinOptions) logf(format string, args ...any) {+ if o.Log == nil {+ return+ }+ _, _ = fmt.Fprintf(o.Log, time.Now().UTC().Format(time.RFC3339)+" "+format+"\n", args...)+}++func newMember(opts JoinOptions, i int) *member {+ dir := filepath.Join(opts.Dir, fmt.Sprintf("member-%04d", i))+ metricsAddr := fmt.Sprintf("127.0.0.1:%d", opts.MetricsBasePort+i)+ return &member{+ index: i,+ dir: dir,+ socket: filepath.Join(dir, "node.sock"),+ metricsAddr: metricsAddr,+ readyz: "http://" + metricsAddr + "/readyz",+ logPath: filepath.Join(dir, "node.log"),+ }+}++// start launches the member's sam-node. Its p2p listeners take ephemeral+// ports: the fixed defaults would collide across members, and a member is+// reached through the router's relay regardless of where it listens.+func (m *member) start(opts JoinOptions) error {+ if err := os.MkdirAll(m.dir, 0o700); err != nil {+ return err+ }+ logFile, err := os.Create(m.logPath)+ if err != nil {+ return err+ }+ args := []string{+ "run",+ "--control-plane", opts.ControlPlane,+ "--config", filepath.Join(m.dir, "sam-node.yaml"),+ "--data-dir", m.dir,+ "--socket-path", m.socket,+ "--bind-addr=",+ "--metrics-addr", m.metricsAddr,+ "--listen", "/ip4/0.0.0.0/tcp/0",+ "--listen", "/ip4/0.0.0.0/udp/0/quic-v1",+ }+ if opts.BootstrapTokenPath != "" {+ args = append(args, "--bootstrap-token-path", opts.BootstrapTokenPath)+ }+ args = append(args, opts.NodeArgs...)++ cmd := exec.Command(opts.NodeBin, args...) // #nosec G204 -- the operator names the binary and its arguments+ cmd.Stdout = logFile+ cmd.Stderr = logFile+ m.cmd = cmd+ m.logFile = logFile+ m.exited = make(chan struct{})+ m.started = time.Now()+ if err := cmd.Start(); err != nil {+ _ = logFile.Close()+ m.cmd = nil+ m.exited = nil+ return err+ }+ go func() {+ m.exitErr = cmd.Wait()+ close(m.exited)+ }()+ return nil+}++func journey(ctx context.Context, opts JoinOptions, m *member) {+ if err := m.waitReady(ctx, opts.ReadyTimeout); err != nil {+ opts.logf("member %04d failed: %s: %s", m.index, m.stage, m.detail)+ return+ }+ if opts.Service == "" {+ opts.logf("member %04d ready in %.1fs", m.index, m.readyAt.Sub(m.started).Seconds())+ return+ }+ if err := m.firstCall(ctx, opts); err != nil {+ opts.logf("member %04d ready in %.1fs, then failed: %s: %s", m.index, m.readyAt.Sub(m.started).Seconds(), m.stage, m.detail)+ return+ }+ opts.logf("member %04d ready in %.1fs, called %s via %s after %d attempt(s) in %.1fs",+ m.index, m.readyAt.Sub(m.started).Seconds(), opts.Service, m.provider, m.attempts, m.calledAt.Sub(m.readyAt).Seconds())+}++// waitReady polls the member's /readyz, which answers 200 only once the node+// holds an authenticated connection to a router.+func (m *member) waitReady(ctx context.Context, timeout time.Duration) error {+ client := &http.Client{Timeout: 2 * time.Second}+ deadline := time.Now().Add(timeout)+ interval := readyPollMin+ for {+ if isReady(ctx, client, m.readyz) {+ m.readyAt = time.Now()+ return nil+ }+ if time.Now().After(deadline) {+ m.fail(stageReady, fmt.Sprintf("no /readyz 200 within %s (see %s)", timeout, m.logPath))+ return errors.New(m.stage)+ }+ select {+ case <-ctx.Done():+ m.fail(stageCanceled, "while waiting for readiness")+ return ctx.Err()+ case <-m.exited:+ m.fail(stageExited, fmt.Sprintf("%v (see %s)", m.exitErr, m.logPath))+ return errors.New(m.stage)+ case <-time.After(interval):+ }+ interval = min(interval*3/2, readyPollMax)+ }+}++func isReady(ctx context.Context, client *http.Client, url string) bool {+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)+ if err != nil {+ return false+ }+ resp, err := client.Do(req)+ if err != nil {+ return false+ }+ _, _ = io.Copy(io.Discard, resp.Body)+ _ = resp.Body.Close()+ return resp.StatusCode == http.StatusOK+}++// firstCall finds the service and speaks MCP to one of its providers through+// the mesh. Every provider is tried and the whole thing retried until the+// deadline: a provider record can name a peer a rollout just replaced, and a+// member that joined seconds ago has a thin routing table.+func (m *member) firstCall(ctx context.Context, opts JoinOptions) error {+ client := unixClient(m.socket)+ defer client.CloseIdleConnections()+ deadline := m.readyAt.Add(opts.CallTimeout)+ lastErr := "none discovered"+ for {+ providers, err := discover(ctx, client, opts.Service)+ if err != nil {+ lastErr = err.Error()+ }+ for _, p := range providers {+ m.attempts+++ if err := initialize(ctx, client, p, opts.Service); err != nil {+ lastErr = p + ": " + err.Error()+ if m.callErrors == nil {+ m.callErrors = map[string]int{}+ }+ m.callErrors[err.Error()]+++ continue+ }+ m.calledAt = time.Now()+ m.provider = p+ return nil+ }+ if time.Now().After(deadline) {+ m.fail(stageCall, fmt.Sprintf("%s after %d attempt(s) within %s; last: %s", opts.Service, m.attempts, opts.CallTimeout, lastErr))+ return errors.New(m.stage)+ }+ select {+ case <-ctx.Done():+ m.fail(stageCanceled, "while calling "+opts.Service)+ return ctx.Err()+ case <-m.exited:+ m.fail(stageExited, fmt.Sprintf("%v (see %s)", m.exitErr, m.logPath))+ return errors.New(m.stage)+ case <-time.After(callRetryInterval):+ }+ }+}++func unixClient(socket string) *http.Client {+ return &http.Client{Transport: &http.Transport{+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {+ return (&net.Dialer{}).DialContext(ctx, "unix", socket)+ },+ }}+}++func discover(ctx context.Context, client *http.Client, service string) ([]string, error) {+ ctx, cancel := context.WithTimeout(ctx, requestTimeout)+ defer cancel()+ url := "http://localhost/sam/service/discover?type=mcp&name=" + service + "&timeout=20s"+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)+ if err != nil {+ return nil, err+ }+ resp, err := client.Do(req)+ if err != nil {+ return nil, fmt.Errorf("discover: %w", err)+ }+ defer func() { _ = resp.Body.Close() }()+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyProbe))+ if err != nil {+ return nil, fmt.Errorf("discover: %w", err)+ }+ if resp.StatusCode != http.StatusOK {+ return nil, fmt.Errorf("discover: HTTP %d: %s", resp.StatusCode, head(body))+ }+ var providers []*api.DiscoveredProvider+ if err := json.Unmarshal(body, &providers); err != nil {+ return nil, fmt.Errorf("discover: %w", err)+ }+ peers := make([]string, 0, len(providers))+ for _, p := range providers {+ if p.GetPeerId() != "" {+ peers = append(peers, p.GetPeerId())+ }+ }+ return peers, nil+}++// initialize opens an MCP session with one provider through the mesh. The+// answer is JSON or an SSE frame; either names the server, and the body is+// read only as far as that name. A JSON-RPC error arrives as HTTP 200 and is+// reported by its message.+func initialize(ctx context.Context, client *http.Client, peer, service string) error {+ ctx, cancel := context.WithTimeout(ctx, requestTimeout)+ defer cancel()+ url := "http://localhost/sam/" + peer + "/mcp/" + service+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(mcpInitialize))+ if err != nil {+ return err+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Accept", "application/json, text/event-stream")+ resp, err := client.Do(req)+ if err != nil {+ return err+ }+ defer func() { _ = resp.Body.Close() }()++ var body []byte+ chunk := make([]byte, 4096)+ for len(body) < maxBodyProbe {+ n, err := resp.Body.Read(chunk)+ body = append(body, chunk[:n]...)+ if resp.StatusCode == http.StatusOK && bytes.Contains(body, []byte(`"serverInfo"`)) {+ return nil+ }+ if err != nil {+ break+ }+ }+ if msg := rpcErrorMessage(body); msg != "" {+ return fmt.Errorf("HTTP %d, JSON-RPC error: %s", resp.StatusCode, msg)+ }+ return fmt.Errorf("HTTP %d without serverInfo: %s", resp.StatusCode, head(body))+}++// rpcErrorMessage extracts the message of a JSON-RPC error response, or the+// empty string when body is not one.+func rpcErrorMessage(body []byte) string {+ var rpc struct {+ Error *struct {+ Message string `json:"message"`+ } `json:"error"`+ }+ if json.Unmarshal(bytes.TrimSpace(body), &rpc) != nil || rpc.Error == nil {+ return ""+ }+ return rpc.Error.Message+}++func head(body []byte) string {+ s := strings.TrimSpace(string(body))+ if len(s) > 120 {+ s = s[:120] + "..."+ }+ return s+}++func summariseJoin(opts JoinOptions, members []*member, elapsed time.Duration) *JoinReport {+ report := &JoinReport{+ Count: opts.Count,+ Ramp: opts.Ramp,+ Service: opts.Service,+ Elapsed: elapsed.Seconds(),+ Errors: map[string]int{},+ CallErrors: map[string]int{},+ Members: make([]MemberResult, 0, len(members)),+ }+ var ready, call, journey []time.Duration+ for _, m := range members {+ report.Members = append(report.Members, m.result())+ report.CallAttempts += m.attempts+ for msg, n := range m.callErrors {+ report.CallErrors[msg] += n+ }+ if !m.readyAt.IsZero() {+ report.Ready+++ ready = append(ready, m.readyAt.Sub(m.started))+ }+ if !m.calledAt.IsZero() {+ call = append(call, m.calledAt.Sub(m.readyAt))+ journey = append(journey, m.calledAt.Sub(m.started))+ }+ switch {+ case m.stage != "":+ report.Failed+++ report.Errors[m.stage]+++ default:+ report.Completed+++ }+ }+ report.ReadyMs = summarise(ready)+ report.FirstCallMs = summarise(call)+ report.JourneyMs = summarise(journey)+ return report+}++// hold samples every member's readiness until the hold elapses or the+// context is cancelled, and turns the transitions into outage windows.+func hold(ctx context.Context, opts JoinOptions, members []*member) *HoldReport {+ type state struct {+ ready bool+ downSince time.Time+ flapped bool+ }+ states := make([]state, len(members))+ report := &HoldReport{}+ var outages []time.Duration++ client := &http.Client{Timeout: 2 * time.Second}+ start := time.Now()+ end := start.Add(opts.Hold)+ for {+ now := time.Now()+ ready := sampleReady(ctx, client, members)+ count := 0+ for i, m := range members {+ if ready[i] {+ count+++ }+ s := &states[i]+ switch {+ case report.Samples == 0:+ s.ready = ready[i]+ if !ready[i] {+ s.downSince = now+ }+ case s.ready && !ready[i]:+ s.ready = false+ s.downSince = now+ s.flapped = true+ report.Flaps+++ opts.logf("member %04d lost its router", m.index)+ case !s.ready && ready[i]:+ s.ready = true+ outages = append(outages, now.Sub(s.downSince))+ opts.logf("member %04d back after %.1fs", m.index, now.Sub(s.downSince).Seconds())+ }+ }+ report.Series = append(report.Series, ReadySample{At: now, Ready: count})+ if report.Samples == 0 {+ report.Resident = count+ report.MinReady = count+ } else if count < report.MinReady {+ report.MinReady = count+ }+ report.Samples++++ if now.After(end) || ctx.Err() != nil {+ break+ }+ select {+ case <-ctx.Done():+ case <-time.After(opts.SampleInterval):+ }+ if ctx.Err() != nil {+ break+ }+ }++ report.Seconds = time.Since(start).Seconds()+ report.OutageMs = summarise(outages)+ for i, m := range members {+ if states[i].flapped {+ report.Flapped+++ }+ if !states[i].ready {+ report.NotReadyAtEnd+++ }+ if m.cmd != nil && m.hasExited() {+ report.Exited+++ }+ }+ return report+}++// sampleReady reads every member's /readyz, a bounded number at a time.+func sampleReady(ctx context.Context, client *http.Client, members []*member) []bool {+ ready := make([]bool, len(members))+ sem := make(chan struct{}, sampleConcurrency)+ var wg sync.WaitGroup+ for i, m := range members {+ if m.cmd == nil || m.hasExited() {+ continue+ }+ wg.Add(1)+ sem <- struct{}{}+ go func() {+ defer wg.Done()+ defer func() { <-sem }()+ ready[i] = isReady(ctx, client, m.readyz)+ }()+ }+ wg.Wait()+ return ready+}++// teardown stops every member: a SIGTERM, then a SIGKILL for whatever is+// still running when the grace period ends.+func teardown(members []*member) {+ for _, m := range members {+ if m.cmd != nil && !m.hasExited() {+ _ = m.cmd.Process.Signal(syscall.SIGTERM)+ }+ }+ deadline := time.Now().Add(teardownGrace)+ for _, m := range members {+ if m.cmd == nil {+ continue+ }+ select {+ case <-m.exited:+ case <-time.After(time.Until(deadline)):+ _ = m.cmd.Process.Kill()+ <-m.exited+ }+ _ = m.logFile.Close()+ }+}++func millis(d time.Duration) float64 {+ return float64(d.Nanoseconds()) / 1e6+}diff --git a/internal/bench/join_test.go b/internal/bench/join_test.gonew file mode 100644index 00000000..20804973--- /dev/null+++ b/internal/bench/join_test.go@@ -0,0 +1,379 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package bench++import (+ "context"+ "encoding/json"+ "fmt"+ "net"+ "net/http"+ "os"+ "os/signal"+ "path/filepath"+ "strings"+ "syscall"+ "testing"+ "time"+)++// The members RunJoin starts are whatever binary it is pointed at. Here that+// is this test binary again, re-entered through TestMain as a node that+// answers /readyz and the two sidecar routes the journey uses, with its+// behaviour set through the environment the children inherit.++func TestMain(m *testing.M) {+ if os.Getenv("SAM_BENCH_FAKE_NODE") == "1" {+ os.Exit(fakeNode(os.Args[1:]))+ }+ os.Exit(m.Run())+}++func fakeNode(args []string) int {+ var metricsAddr, socket, tokenPath string+ for i := 0; i < len(args)-1; i++ {+ switch args[i] {+ case "--metrics-addr":+ metricsAddr = args[i+1]+ case "--socket-path":+ socket = args[i+1]+ case "--bootstrap-token-path":+ tokenPath = args[i+1]+ }+ }+ if os.Getenv("FAKE_NODE_EXIT") != "" {+ fmt.Fprintln(os.Stderr, "fake node: refusing to start")+ return 3+ }+ if tokenPath == "" {+ fmt.Fprintln(os.Stderr, "fake node: no --bootstrap-token-path")+ return 2+ }++ readyAfter, _ := time.ParseDuration(os.Getenv("FAKE_NODE_READY_AFTER"))+ flapAt, _ := time.ParseDuration(os.Getenv("FAKE_NODE_FLAP_AT"))+ flapFor, _ := time.ParseDuration(os.Getenv("FAKE_NODE_FLAP_FOR"))+ providers := strings.Split(os.Getenv("FAKE_NODE_PROVIDERS"), ",")+ start := time.Now()+ ready := func() bool {+ elapsed := time.Since(start)+ if elapsed < readyAfter {+ return false+ }+ return flapAt == 0 || elapsed < flapAt || elapsed >= flapAt+flapFor+ }++ metrics := http.NewServeMux()+ metrics.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {+ if !ready() {+ http.Error(w, "not ready", http.StatusServiceUnavailable)+ return+ }+ w.WriteHeader(http.StatusOK)+ })+ sidecar := http.NewServeMux()+ sidecar.HandleFunc("/sam/service/discover", func(w http.ResponseWriter, _ *http.Request) {+ var out []map[string]string+ for _, p := range providers {+ if p != "" {+ out = append(out, map[string]string{"peer_id": p})+ }+ }+ _ = json.NewEncoder(w).Encode(out)+ })+ sidecar.HandleFunc("/sam/", func(w http.ResponseWriter, r *http.Request) {+ if strings.HasPrefix(r.URL.Path, "/sam/dead/") {+ http.Error(w, "no route to peer", http.StatusBadGateway)+ return+ }+ _, _ = fmt.Fprint(w, `{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-06-18","serverInfo":{"name":"fake","version":"0"}}}`)+ })++ tcp, err := net.Listen("tcp", metricsAddr)+ if err != nil {+ fmt.Fprintln(os.Stderr, "fake node:", err)+ return 1+ }+ unix, err := net.Listen("unix", socket)+ if err != nil {+ fmt.Fprintln(os.Stderr, "fake node:", err)+ return 1+ }+ go func() { _ = http.Serve(tcp, metrics) }() //nolint:gosec // a test double+ go func() { _ = http.Serve(unix, sidecar) }() //nolint:gosec // a test double+ stop := make(chan os.Signal, 1)+ signal.Notify(stop, syscall.SIGTERM, os.Interrupt)+ <-stop+ return 0+}++func fakeFleet(t *testing.T, count int, env map[string]string) JoinOptions {+ t.Helper()+ t.Setenv("SAM_BENCH_FAKE_NODE", "1")+ if env == nil {+ env = map[string]string{}+ }+ if _, ok := env["FAKE_NODE_PROVIDERS"]; !ok {+ env["FAKE_NODE_PROVIDERS"] = "alive"+ }+ for _, k := range []string{"FAKE_NODE_EXIT", "FAKE_NODE_READY_AFTER", "FAKE_NODE_FLAP_AT", "FAKE_NODE_FLAP_FOR", "FAKE_NODE_PROVIDERS"} {+ t.Setenv(k, env[k])+ }+ token := filepath.Join(t.TempDir(), "token")+ if err := os.WriteFile(token, []byte("sam-bt-test\n"), 0o600); err != nil {+ t.Fatal(err)+ }+ // t.TempDir carries the test's name, which for a subtest is long enough+ // to push a member's socket past the Unix path limit.+ dir, err := os.MkdirTemp("", "sam-join-")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = os.RemoveAll(dir) })+ return JoinOptions{+ NodeBin: os.Args[0],+ Count: count,+ Dir: filepath.Join(dir, "fleet"),+ ControlPlane: "http://127.0.0.1:1",+ BootstrapTokenPath: token,+ Service: "everything",+ MetricsBasePort: freePortBase(t, count),+ }+}++// freePortBase finds count consecutive free loopback ports.+func freePortBase(t *testing.T, count int) int {+ t.Helper()+ for attempt := 0; attempt < 20; attempt++ {+ l, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatal(err)+ }+ base := l.Addr().(*net.TCPAddr).Port+ _ = l.Close()+ free := true+ for i := 1; i < count; i++ {+ probe, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i))+ if err != nil {+ free = false+ break+ }+ _ = probe.Close()+ }+ if free {+ return base+ }+ }+ t.Fatal("no run of free ports")+ return 0+}++func TestRunJoinTimesEachStageAndRetriesStaleProviders(t *testing.T) {+ // A provider record that names a peer which does not answer is the+ // normal state of a mesh just after a rollout. The journey has to get+ // past it and the report has to say that it happened.+ opts := fakeFleet(t, 3, map[string]string{+ "FAKE_NODE_READY_AFTER": "200ms",+ "FAKE_NODE_PROVIDERS": "dead,alive",+ })++ report, err := RunJoin(context.Background(), opts)+ if err != nil {+ t.Fatalf("RunJoin: %v", err)+ }+ if report.Ready != 3 || report.Completed != 3 || report.Failed != 0 {+ t.Fatalf("ready/completed/failed = %d/%d/%d, want 3/3/0 (errors %v)", report.Ready, report.Completed, report.Failed, report.Errors)+ }+ if report.ReadyMs.Count != 3 || report.ReadyMs.Min < 200 {+ t.Errorf("ready_ms = %+v, want 3 samples none under the 200ms the node took", report.ReadyMs)+ }+ if report.JourneyMs.Count != 3 || report.FirstCallMs.Count != 3 {+ t.Errorf("journey/first-call counts = %d/%d, want 3/3", report.JourneyMs.Count, report.FirstCallMs.Count)+ }+ if report.CallAttempts != 6 {+ t.Errorf("call_attempts = %d, want 6: the dead provider tried once per member before the live one", report.CallAttempts)+ }+ if len(report.CallErrors) != 1 {+ t.Errorf("call_errors = %v, want the dead provider's refusal once per member", report.CallErrors)+ }+ for msg, n := range report.CallErrors {+ if n != 3 || !strings.Contains(msg, "HTTP 502") {+ t.Errorf("call_errors = %v, want 3 x HTTP 502", report.CallErrors)+ }+ }+ for _, m := range report.Members {+ if m.Provider != "alive" || m.Attempts != 2 || m.Error != "" {+ t.Errorf("member %d = %+v, want provider alive after 2 attempts", m.Index, m)+ }+ if _, err := os.Stat(filepath.Join(m.Dir, "node.log")); err != nil {+ t.Errorf("member %d kept no log: %v", m.Index, err)+ }+ }+ if report.Hold != nil {+ t.Errorf("hold reported without --hold")+ }+}++func TestRunJoinReportsMembersLeftBehindByStage(t *testing.T) {+ // A fleet that mostly failed must not read as a fast one: a member that+ // never got there is counted, by the stage it stopped at, and never+ // contributes a latency.+ t.Run("process exits", func(t *testing.T) {+ opts := fakeFleet(t, 2, map[string]string{"FAKE_NODE_EXIT": "1"})+ report, err := RunJoin(context.Background(), opts)+ if err != nil {+ t.Fatalf("RunJoin: %v", err)+ }+ if report.Failed != 2 || report.Errors[stageExited] != 2 {+ t.Fatalf("failed = %d, errors = %v, want 2 under %q", report.Failed, report.Errors, stageExited)+ }+ if report.ReadyMs.Count != 0 {+ t.Errorf("a member that exited contributed a readiness latency")+ }+ if !strings.Contains(report.Members[0].Error, "exit status 3") {+ t.Errorf("member error = %q, want the exit status", report.Members[0].Error)+ }+ })+ t.Run("never ready", func(t *testing.T) {+ opts := fakeFleet(t, 2, map[string]string{"FAKE_NODE_READY_AFTER": "1h"})+ opts.ReadyTimeout = 300 * time.Millisecond+ report, err := RunJoin(context.Background(), opts)+ if err != nil {+ t.Fatalf("RunJoin: %v", err)+ }+ if report.Failed != 2 || report.Errors[stageReady] != 2 {+ t.Fatalf("failed = %d, errors = %v, want 2 under %q", report.Failed, report.Errors, stageReady)+ }+ })+ t.Run("no provider answers", func(t *testing.T) {+ opts := fakeFleet(t, 1, map[string]string{"FAKE_NODE_PROVIDERS": "dead"})+ opts.CallTimeout = 100 * time.Millisecond+ report, err := RunJoin(context.Background(), opts)+ if err != nil {+ t.Fatalf("RunJoin: %v", err)+ }+ if report.Ready != 1 || report.Completed != 0 || report.Errors[stageCall] != 1 {+ t.Fatalf("ready/completed = %d/%d, errors = %v, want 1/0 under %q", report.Ready, report.Completed, report.Errors, stageCall)+ }+ if report.ReadyMs.Count != 1 || report.JourneyMs.Count != 0 {+ t.Errorf("ready/journey counts = %d/%d: readiness was reached, the journey was not", report.ReadyMs.Count, report.JourneyMs.Count)+ }+ })+}++func TestRunJoinHoldTurnsFlapsIntoOutageWindows(t *testing.T) {+ // What a resident fleet is for: a router that goes away for a while+ // shows up as one outage per member with a length, and a fleet that is+ // whole again at the end says so.+ opts := fakeFleet(t, 2, map[string]string{+ "FAKE_NODE_FLAP_AT": "1200ms",+ "FAKE_NODE_FLAP_FOR": "400ms",+ })+ opts.Hold = 2 * time.Second+ opts.SampleInterval = 100 * time.Millisecond+ var residentCompleted int+ var residentHadHold bool+ opts.OnResident = func(r *JoinReport) { residentCompleted, residentHadHold = r.Completed, r.Hold != nil }++ report, err := RunJoin(context.Background(), opts)+ if err != nil {+ t.Fatalf("RunJoin: %v", err)+ }+ if report.Completed != 2 {+ t.Fatalf("completed = %d, want 2 (errors %v)", report.Completed, report.Errors)+ }+ if residentCompleted != 2 || residentHadHold {+ t.Errorf("OnResident saw completed=%d hold=%v, want the journey report before the hold", residentCompleted, residentHadHold)+ }+ h := report.Hold+ if h == nil {+ t.Fatal("no hold report")+ }+ if h.Resident != 2 || h.MinReady != 0 {+ t.Errorf("resident/min_ready = %d/%d, want 2/0", h.Resident, h.MinReady)+ }+ if h.Flaps != 2 || h.Flapped != 2 || h.OutageMs.Count != 2 {+ t.Errorf("flaps/flapped/outages = %d/%d/%d, want 2/2/2", h.Flaps, h.Flapped, h.OutageMs.Count)+ }+ if h.OutageMs.Min < 300 || h.OutageMs.Max > 900 {+ t.Errorf("outage_ms = %+v, want windows around the 400ms the node was away", h.OutageMs)+ }+ if h.NotReadyAtEnd != 0 || h.Exited != 0 {+ t.Errorf("not_ready_at_end/exited = %d/%d, want 0/0", h.NotReadyAtEnd, h.Exited)+ }+ if h.Samples < 10 || len(h.Series) != h.Samples {+ t.Errorf("samples = %d, series = %d, want at least 10 and equal", h.Samples, len(h.Series))+ }+}++func TestRunJoinCancelEndsTheHoldWithAReport(t *testing.T) {+ // Ctrl-C during a long hold is how most holds end. The report is the+ // point of the run and must survive it.+ opts := fakeFleet(t, 1, nil)+ opts.Hold = time.Hour+ opts.SampleInterval = 50 * time.Millisecond++ ctx, cancel := context.WithCancel(context.Background())+ go func() {+ time.Sleep(1500 * time.Millisecond)+ cancel()+ }()+ done := make(chan struct{})+ var report *JoinReport+ var err error+ go func() {+ report, err = RunJoin(ctx, opts)+ close(done)+ }()+ select {+ case <-done:+ case <-time.After(15 * time.Second):+ t.Fatal("RunJoin did not return after cancel")+ }+ if err != nil {+ t.Fatalf("RunJoin: %v", err)+ }+ if report.Hold == nil || report.Hold.Samples == 0 {+ t.Fatalf("hold = %+v, want samples from before the cancel", report.Hold)+ }+ if report.Hold.Seconds > 60 {+ t.Errorf("hold ran %.0fs, want it ended by the cancel", report.Hold.Seconds)+ }+}++func TestRunJoinRefusesADirectoryThatHoldsState(t *testing.T) {+ opts := fakeFleet(t, 1, nil)+ if err := os.MkdirAll(filepath.Join(opts.Dir, "member-0000"), 0o700); err != nil {+ t.Fatal(err)+ }+ if _, err := RunJoin(context.Background(), opts); err == nil || !strings.Contains(err.Error(), "not empty") {+ t.Fatalf("err = %v, want a refusal to reuse member state", err)+ }+}++func TestRunJoinRampSpacesTheStarts(t *testing.T) {+ opts := fakeFleet(t, 3, nil)+ opts.Ramp = 5+ report, err := RunJoin(context.Background(), opts)+ if err != nil {+ t.Fatalf("RunJoin: %v", err)+ }+ if report.Elapsed < 0.4 {+ t.Errorf("elapsed = %.2fs, want at least the 0.4s two gaps at 5/s take", report.Elapsed)+ }+ if report.Completed != 3 {+ t.Errorf("completed = %d, want 3 (errors %v)", report.Completed, report.Errors)+ }+}