Skip to content

Agent Mesh

bench: add sam-bench join, the journey of the Nth member

Antonio Ojeacd5d80ad5ca1
diff --git a/cmd/sam-bench/main.go b/cmd/sam-bench/main.go
index a04d4bf3..242b6ad9 100644
--- a/cmd/sam-bench/main.go
+++ b/cmd/sam-bench/main.go
@@ -150,7 +150,7 @@ func main() {
if err := runCmd.MarkFlagRequired("target"); err != nil {
panic(err)
}
- rootCmd.AddCommand(runCmd, newReportCmd(), newSTSCmd())
+ rootCmd.AddCommand(runCmd, newReportCmd(), newSTSCmd(), newJoinCmd())
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
@@ -197,6 +197,102 @@ func newSTSCmd() *cobra.Command {
return cmd
}
+// joinObservation is one fleet's journey, with what the mesh's own processes
+// reported before and after it, in the same shape as an observation.
+type joinObservation struct {
+ Labels map[string]string `json:"labels,omitempty"`
+ Started time.Time `json:"started"`
+ Join *bench.JoinReport `json:"join"`
+ Before map[string]map[string]float64 `json:"metrics_before,omitempty"`
+ After map[string]map[string]float64 `json:"metrics_after,omitempty"`
+}
+
+func newJoinCmd() *cobra.Command {
+ var (
+ opts bench.JoinOptions
+ marker string
+ scrape []string
+ labels []string
+ out string
+ )
+ cmd := &cobra.Command{
+ Use: "join",
+ Short: "Start a fleet of members and record what joining cost each of them",
+ Long: "Starts N sam-node processes from one bootstrap token and times each one from start\n" +
+ "to an authenticated router connection to its first call of a service through the\n" +
+ "mesh. With --hold the fleet then stays resident and every member's readiness is\n" +
+ "sampled, so a router rollout or a key rotation that happens meanwhile shows up as\n" +
+ "outage windows. The processes all share this host's source address.",
+ SilenceUsage: true,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ tags, err := parseLabels(labels)
+ if err != nil {
+ return err
+ }
+ if opts.Dir == "" {
+ if opts.Dir, err = os.MkdirTemp("", "sam-join-"); err != nil {
+ return err
+ }
+ }
+ opts.Log = os.Stderr
+
+ obs := joinObservation{Labels: tags, Started: time.Now()}
+ // A hold can run for a day; the join numbers are written as soon
+ // as they exist and the marker tells a script the fleet is up.
+ opts.OnResident = func(r *bench.JoinReport) {
+ obs.Join = r
+ if out != "" {
+ if err := write(out, obs); err != nil {
+ fmt.Fprintf(os.Stderr, "sam-bench: %v\n", err)
+ }
+ }
+ if marker != "" {
+ stamp := []byte(time.Now().UTC().Format(time.RFC3339) + "\n")
+ if err := os.WriteFile(marker, stamp, 0o600); err != nil {
+ fmt.Fprintf(os.Stderr, "sam-bench: %v\n", err)
+ }
+ }
+ }
+ obs.Before, err = scrapeAll(cmd.Context(), scrape)
+ if err != nil {
+ return err
+ }
+ obs.Join, err = bench.RunJoin(cmd.Context(), opts)
+ if err != nil {
+ return err
+ }
+ // The fleet is gone by now; a scrape that fails here would lose the
+ // report over a metrics endpoint, so it is reported and not fatal.
+ if obs.After, err = scrapeAll(context.Background(), scrape); err != nil {
+ fmt.Fprintf(os.Stderr, "sam-bench: metrics after the run: %v\n", err)
+ }
+ return write(out, obs)
+ },
+ }
+ flags := cmd.Flags()
+ flags.StringVar(&opts.NodeBin, "node-bin", "sam-node", "sam-node binary to start one process of per member")
+ flags.IntVar(&opts.Count, "count", 10, "Members to start")
+ flags.StringVar(&opts.Dir, "dir", "", "Empty directory for member state, sockets and logs; default a new temporary one")
+ flags.StringVar(&opts.ControlPlane, "control-plane", "", "Control plane URL every member enrolls with (required)")
+ flags.StringVar(&opts.BootstrapTokenPath, "bootstrap-token-path", "", "File holding the bootstrap token the members enroll with; it needs --count usages left")
+ flags.StringVar(&opts.Service, "service", "everything", "MCP service each member finds and calls once ready; empty ends the journey at readiness")
+ flags.Float64Var(&opts.Ramp, "ramp", 0, "Members started per second; 0 starts them all at once")
+ flags.DurationVar(&opts.ReadyTimeout, "ready-timeout", 180*time.Second, "Bound on one member's enrollment and router authentication")
+ flags.DurationVar(&opts.CallTimeout, "call-timeout", 120*time.Second, "Bound on one member's discovery and first call, from readiness")
+ flags.DurationVar(&opts.Hold, "hold", 0, "Keep the fleet resident this long after the journeys end, sampling readiness; 0 tears it down at once")
+ flags.DurationVar(&opts.SampleInterval, "sample-interval", 10*time.Second, "How often each resident member's readiness is read during --hold")
+ flags.IntVar(&opts.MetricsBasePort, "metrics-base-port", 20000, "Member i serves /readyz on 127.0.0.1:(port+i)")
+ flags.StringArrayVar(&opts.NodeArgs, "node-arg", nil, "Extra argument for every sam-node, repeatable (e.g. --node-arg=--insecure-control-plane)")
+ flags.StringVar(&marker, "resident-marker", "", "File to create once the fleet is resident and the hold has begun")
+ flags.StringArrayVar(&scrape, "scrape", nil, "Metrics endpoint to record before and after the run, repeatable")
+ flags.StringArrayVar(&labels, "label", nil, "Condition to record with the observation as name=value, repeatable")
+ flags.StringVar(&out, "out", "", "File to write the observation to; default stdout")
+ if err := cmd.MarkFlagRequired("control-plane"); err != nil {
+ panic(err)
+ }
+ return cmd
+}
+
// scrapeAll records every endpoint, refusing to continue if one is missing:
// an observation with a hole in it is worse than no observation, because it
// still looks like data.
@@ -246,7 +342,7 @@ func parseLabels(raw []string) (map[string]string, error) {
return labels, nil
}
-func write(path string, obs observation) error {
+func write(path string, obs any) error {
encoded, err := json.MarshalIndent(obs, "", " ")
if err != nil {
return err
diff --git a/internal/bench/join.go b/internal/bench/join.go
new file mode 100644
index 00000000..32e61164
--- /dev/null
+++ b/internal/bench/join.go
@@ -0,0 +1,807 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bench
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+ "syscall"
+ "time"
+
+ "github.com/google/sam/api"
+)
+
+// A density run says how many agents one member can carry. This asks the
+// other question: what the Nth member experiences while N-1 are already
+// there. The journey measured is the one a new user takes: start a node with
+// nothing but a bootstrap token, wait until it holds an authenticated router
+// connection, find a service someone else advertised and make one call to it
+// through the mesh. Each step is timed per member, so a burst of joiners that
+// leaves a few behind is reported as a few left behind rather than as a
+// slightly worse mean.
+//
+// The members are real sam-node processes, one per member, started from this
+// process so that every timestamp comes from one clock. They all share this
+// host's source address, which is what a classroom, an office or a CI runner
+// pool looks like from the router.
+
+// JoinOptions describe one fleet to take through the join journey.
+type JoinOptions struct {
+ // NodeBin is the sam-node binary; one process of it is started per member.
+ NodeBin string
+
+ // Count is how many members to start.
+ Count int
+
+ // Dir is the directory under which each member gets its own state,
+ // socket and log. It must be empty: a member directory that already holds
+ // an identity would resume rather than join.
+ Dir string
+
+ // ControlPlane is the URL every member enrolls with.
+ ControlPlane string
+
+ // BootstrapTokenPath is the file holding the bootstrap token the members
+ // enroll with. It needs at least Count usages left. Empty means the
+ // credential comes from NodeArgs instead, such as a --jwt-path.
+ BootstrapTokenPath string
+
+ // Service is the MCP service each member looks up and calls once it is
+ // ready. Empty ends the journey at readiness.
+ Service string
+
+ // Ramp is how many members to start per second. Zero starts them all at
+ // once.
+ Ramp float64
+
+ // ReadyTimeout bounds enrollment plus router authentication for one
+ // member.
+ ReadyTimeout time.Duration
+
+ // CallTimeout bounds discovery plus the first successful call for one
+ // member, counted from when it became ready.
+ CallTimeout time.Duration
+
+ // Hold keeps the fleet resident after the last journey has ended and
+ // samples every member's readiness until it elapses or the context is
+ // cancelled. Zero tears the fleet down as soon as the journeys end.
+ Hold time.Duration
+
+ // SampleInterval is how often each resident member is read during Hold.
+ SampleInterval time.Duration
+
+ // MetricsBasePort is the first of Count consecutive loopback ports;
+ // member i serves /readyz on MetricsBasePort+i.
+ MetricsBasePort int
+
+ // NodeArgs are appended to every member's command line as given.
+ NodeArgs []string
+
+ // OnResident, if set, is called with the report once the journeys have
+ // ended and the hold has begun. The report is complete except for Hold,
+ // so a caller can publish the join numbers before a long hold ends.
+ OnResident func(*JoinReport)
+
+ // Log receives one line per event: a member's journey ending, a resident
+ // member losing or regaining its router. Nil discards them.
+ Log io.Writer
+}
+
+// JoinReport is the outcome of one fleet's journey.
+type JoinReport struct {
+ Count int `json:"count"`
+ Ramp float64 `json:"ramp_per_second"`
+ Service string `json:"service,omitempty"`
+
+ // Ready counts members that reached /readyz 200. Completed counts those
+ // that finished the whole journey, which is readiness alone when there
+ // is no Service. Failed is the rest.
+ Ready int `json:"ready"`
+ Completed int `json:"completed"`
+ Failed int `json:"failed"`
+
+ // Elapsed runs from the first member starting to the last journey ending.
+ Elapsed float64 `json:"elapsed_seconds"`
+
+ // ReadyMs is process start to /readyz 200, FirstCallMs is readiness to
+ // the first successful call, and JourneyMs is their sum, each over the
+ // members that got that far.
+ ReadyMs Distribution `json:"ready_ms"`
+ FirstCallMs Distribution `json:"first_call_ms"`
+ JourneyMs Distribution `json:"journey_ms"`
+
+ // CallAttempts is the number of provider calls made across the fleet.
+ // More than one per completed member means provider records that named
+ // a peer which did not answer; CallErrors says how each of those
+ // attempts failed, including on members that then succeeded elsewhere.
+ CallAttempts int `json:"call_attempts"`
+ CallErrors map[string]int `json:"call_errors,omitempty"`
+
+ // Errors counts failures by stage, so a fleet that mostly failed cannot
+ // be mistaken for a fast one.
+ Errors map[string]int `json:"errors,omitempty"`
+
+ Hold *HoldReport `json:"hold,omitempty"`
+
+ Members []MemberResult `json:"members"`
+}
+
+// MemberResult is one member's journey.
+type MemberResult struct {
+ Index int `json:"index"`
+ Dir string `json:"dir"`
+ ReadyMs float64 `json:"ready_ms,omitempty"`
+ FirstCallMs float64 `json:"first_call_ms,omitempty"`
+ Provider string `json:"provider,omitempty"`
+ Attempts int `json:"call_attempts,omitempty"`
+ Error string `json:"error,omitempty"`
+}
+
+// HoldReport describes a resident fleet over time.
+type HoldReport struct {
+ Seconds float64 `json:"seconds"`
+ Samples int `json:"samples"`
+
+ // Resident is how many members were ready at the first sample and
+ // MinReady the fewest at any sample.
+ Resident int `json:"resident"`
+ MinReady int `json:"min_ready"`
+
+ // Flaps counts ready-to-not-ready transitions across the fleet, Flapped
+ // the members that had at least one, NotReadyAtEnd those still down when
+ // the hold ended and Exited the processes that ended by themselves.
+ Flaps int `json:"flaps"`
+ Flapped int `json:"members_flapped"`
+ NotReadyAtEnd int `json:"members_not_ready_at_end"`
+ Exited int `json:"members_exited"`
+
+ // OutageMs is the length of every not-ready window that ended during
+ // the hold. A window still open at the end is in NotReadyAtEnd instead.
+ OutageMs Distribution `json:"outage_ms"`
+
+ // Series is the ready count at each sample, for lining up against the
+ // events an operator caused while the fleet was resident.
+ Series []ReadySample `json:"ready_series"`
+}
+
+// ReadySample is the fleet's ready count at one instant.
+type ReadySample struct {
+ At time.Time `json:"at"`
+ Ready int `json:"ready"`
+}
+
+// Journey stages, used as the keys of JoinReport.Errors.
+const (
+ stageStart = "start failed"
+ stageReady = "not ready in time"
+ stageExited = "process exited"
+ stageCall = "no provider answered in time"
+ stageCanceled = "cancelled"
+)
+
+const (
+ // Readiness is polled from readyPollMin, backing off to readyPollMax: a
+ // member that is up in a few hundred milliseconds is measured as such,
+ // and one that takes a minute is not asked a thousand times.
+ readyPollMin = 50 * time.Millisecond
+ readyPollMax = 500 * time.Millisecond
+ callRetryInterval = 5 * time.Second
+ requestTimeout = 30 * time.Second
+ teardownGrace = 10 * time.Second
+ sampleConcurrency = 64
+ // maxBodyProbe bounds how much of a response is read looking for the
+ // MCP server's name; an initialize answer is a few hundred bytes.
+ maxBodyProbe = 64 << 10
+)
+
+// mcpInitialize is the first request of an MCP session, enough for the
+// server to name itself.
+const mcpInitialize = `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"sam-bench","version":"0"}}}`
+
+type member struct {
+ index int
+ dir string
+ socket string
+ metricsAddr string
+ readyz string
+ logPath string
+ cmd *exec.Cmd
+ logFile *os.File
+ started time.Time
+ exited chan struct{}
+ exitErr error
+
+ readyAt time.Time
+ calledAt time.Time
+ provider string
+ attempts int
+ callErrors map[string]int
+ stage string
+ detail string
+}
+
+func (m *member) fail(stage, detail string) {
+ m.stage, m.detail = stage, detail
+}
+
+func (m *member) hasExited() bool {
+ if m.exited == nil {
+ return true
+ }
+ select {
+ case <-m.exited:
+ return true
+ default:
+ return false
+ }
+}
+
+func (m *member) result() MemberResult {
+ r := MemberResult{Index: m.index, Dir: m.dir, Provider: m.provider, Attempts: m.attempts}
+ if !m.readyAt.IsZero() {
+ r.ReadyMs = millis(m.readyAt.Sub(m.started))
+ }
+ if !m.calledAt.IsZero() {
+ r.FirstCallMs = millis(m.calledAt.Sub(m.readyAt))
+ }
+ if m.stage != "" {
+ r.Error = m.stage + ": " + m.detail
+ }
+ return r
+}
+
+// RunJoin starts the fleet, takes every member through the journey, holds
+// the fleet if asked, tears it down and reports. Cancelling the context ends
+// whatever phase is running; the report still describes what happened.
+func RunJoin(ctx context.Context, opts JoinOptions) (*JoinReport, error) {
+ if err := opts.validate(); err != nil {
+ return nil, err
+ }
+ opts.setDefaults()
+ if err := os.MkdirAll(opts.Dir, 0o700); err != nil {
+ return nil, fmt.Errorf("bench: %w", err)
+ }
+
+ members := make([]*member, 0, opts.Count)
+ defer func() { teardown(members) }()
+
+ first := time.Now()
+ var wg sync.WaitGroup
+ for i := range opts.Count {
+ if i > 0 && opts.Ramp > 0 {
+ select {
+ case <-time.After(time.Duration(float64(time.Second) / opts.Ramp)):
+ case <-ctx.Done():
+ }
+ }
+ m := newMember(opts, i)
+ members = append(members, m)
+ if ctx.Err() != nil {
+ m.fail(stageCanceled, "before start")
+ continue
+ }
+ if err := m.start(opts); err != nil {
+ m.fail(stageStart, err.Error())
+ continue
+ }
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ journey(ctx, opts, m)
+ }()
+ }
+ wg.Wait()
+
+ report := summariseJoin(opts, members, time.Since(first))
+ if opts.Hold > 0 && ctx.Err() == nil {
+ if opts.OnResident != nil {
+ opts.OnResident(report)
+ }
+ opts.logf("fleet resident: %d of %d ready; holding for %s", report.Ready, report.Count, opts.Hold)
+ report.Hold = hold(ctx, opts, members)
+ }
+ return report, nil
+}
+
+func (o *JoinOptions) validate() error {
+ switch {
+ case o.NodeBin == "":
+ return errors.New("bench: no sam-node binary")
+ case o.Count <= 0:
+ return errors.New("bench: no members to start")
+ case o.Dir == "":
+ return errors.New("bench: no directory for member state")
+ case o.ControlPlane == "":
+ return errors.New("bench: no control plane")
+ case o.Ramp < 0:
+ return errors.New("bench: ramp must not be negative")
+ }
+ entries, err := os.ReadDir(o.Dir)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return fmt.Errorf("bench: %w", err)
+ }
+ if len(entries) > 0 {
+ return fmt.Errorf("bench: %s is not empty; a member directory that already holds an identity would resume rather than join", o.Dir)
+ }
+ // A Unix socket path is limited to 108 bytes on Linux; the member's own
+ // directory and socket name take some of that.
+ if len(filepath.Join(o.Dir, "member-0000", "node.sock")) > 100 {
+ return fmt.Errorf("bench: %s is too deep for a member's socket path", o.Dir)
+ }
+ return nil
+}
+
+func (o *JoinOptions) setDefaults() {
+ if o.ReadyTimeout <= 0 {
+ o.ReadyTimeout = 180 * time.Second
+ }
+ if o.CallTimeout <= 0 {
+ o.CallTimeout = 120 * time.Second
+ }
+ if o.SampleInterval <= 0 {
+ o.SampleInterval = 10 * time.Second
+ }
+ if o.MetricsBasePort <= 0 {
+ o.MetricsBasePort = 20000
+ }
+}
+
+func (o *JoinOptions) logf(format string, args ...any) {
+ if o.Log == nil {
+ return
+ }
+ _, _ = fmt.Fprintf(o.Log, time.Now().UTC().Format(time.RFC3339)+" "+format+"\n", args...)
+}
+
+func newMember(opts JoinOptions, i int) *member {
+ dir := filepath.Join(opts.Dir, fmt.Sprintf("member-%04d", i))
+ metricsAddr := fmt.Sprintf("127.0.0.1:%d", opts.MetricsBasePort+i)
+ return &member{
+ index: i,
+ dir: dir,
+ socket: filepath.Join(dir, "node.sock"),
+ metricsAddr: metricsAddr,
+ readyz: "http://" + metricsAddr + "/readyz",
+ logPath: filepath.Join(dir, "node.log"),
+ }
+}
+
+// start launches the member's sam-node. Its p2p listeners take ephemeral
+// ports: the fixed defaults would collide across members, and a member is
+// reached through the router's relay regardless of where it listens.
+func (m *member) start(opts JoinOptions) error {
+ if err := os.MkdirAll(m.dir, 0o700); err != nil {
+ return err
+ }
+ logFile, err := os.Create(m.logPath)
+ if err != nil {
+ return err
+ }
+ args := []string{
+ "run",
+ "--control-plane", opts.ControlPlane,
+ "--config", filepath.Join(m.dir, "sam-node.yaml"),
+ "--data-dir", m.dir,
+ "--socket-path", m.socket,
+ "--bind-addr=",
+ "--metrics-addr", m.metricsAddr,
+ "--listen", "/ip4/0.0.0.0/tcp/0",
+ "--listen", "/ip4/0.0.0.0/udp/0/quic-v1",
+ }
+ if opts.BootstrapTokenPath != "" {
+ args = append(args, "--bootstrap-token-path", opts.BootstrapTokenPath)
+ }
+ args = append(args, opts.NodeArgs...)
+
+ cmd := exec.Command(opts.NodeBin, args...) // #nosec G204 -- the operator names the binary and its arguments
+ cmd.Stdout = logFile
+ cmd.Stderr = logFile
+ m.cmd = cmd
+ m.logFile = logFile
+ m.exited = make(chan struct{})
+ m.started = time.Now()
+ if err := cmd.Start(); err != nil {
+ _ = logFile.Close()
+ m.cmd = nil
+ m.exited = nil
+ return err
+ }
+ go func() {
+ m.exitErr = cmd.Wait()
+ close(m.exited)
+ }()
+ return nil
+}
+
+func journey(ctx context.Context, opts JoinOptions, m *member) {
+ if err := m.waitReady(ctx, opts.ReadyTimeout); err != nil {
+ opts.logf("member %04d failed: %s: %s", m.index, m.stage, m.detail)
+ return
+ }
+ if opts.Service == "" {
+ opts.logf("member %04d ready in %.1fs", m.index, m.readyAt.Sub(m.started).Seconds())
+ return
+ }
+ if err := m.firstCall(ctx, opts); err != nil {
+ opts.logf("member %04d ready in %.1fs, then failed: %s: %s", m.index, m.readyAt.Sub(m.started).Seconds(), m.stage, m.detail)
+ return
+ }
+ opts.logf("member %04d ready in %.1fs, called %s via %s after %d attempt(s) in %.1fs",
+ m.index, m.readyAt.Sub(m.started).Seconds(), opts.Service, m.provider, m.attempts, m.calledAt.Sub(m.readyAt).Seconds())
+}
+
+// waitReady polls the member's /readyz, which answers 200 only once the node
+// holds an authenticated connection to a router.
+func (m *member) waitReady(ctx context.Context, timeout time.Duration) error {
+ client := &http.Client{Timeout: 2 * time.Second}
+ deadline := time.Now().Add(timeout)
+ interval := readyPollMin
+ for {
+ if isReady(ctx, client, m.readyz) {
+ m.readyAt = time.Now()
+ return nil
+ }
+ if time.Now().After(deadline) {
+ m.fail(stageReady, fmt.Sprintf("no /readyz 200 within %s (see %s)", timeout, m.logPath))
+ return errors.New(m.stage)
+ }
+ select {
+ case <-ctx.Done():
+ m.fail(stageCanceled, "while waiting for readiness")
+ return ctx.Err()
+ case <-m.exited:
+ m.fail(stageExited, fmt.Sprintf("%v (see %s)", m.exitErr, m.logPath))
+ return errors.New(m.stage)
+ case <-time.After(interval):
+ }
+ interval = min(interval*3/2, readyPollMax)
+ }
+}
+
+func isReady(ctx context.Context, client *http.Client, url string) bool {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return false
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return false
+ }
+ _, _ = io.Copy(io.Discard, resp.Body)
+ _ = resp.Body.Close()
+ return resp.StatusCode == http.StatusOK
+}
+
+// firstCall finds the service and speaks MCP to one of its providers through
+// the mesh. Every provider is tried and the whole thing retried until the
+// deadline: a provider record can name a peer a rollout just replaced, and a
+// member that joined seconds ago has a thin routing table.
+func (m *member) firstCall(ctx context.Context, opts JoinOptions) error {
+ client := unixClient(m.socket)
+ defer client.CloseIdleConnections()
+ deadline := m.readyAt.Add(opts.CallTimeout)
+ lastErr := "none discovered"
+ for {
+ providers, err := discover(ctx, client, opts.Service)
+ if err != nil {
+ lastErr = err.Error()
+ }
+ for _, p := range providers {
+ m.attempts++
+ if err := initialize(ctx, client, p, opts.Service); err != nil {
+ lastErr = p + ": " + err.Error()
+ if m.callErrors == nil {
+ m.callErrors = map[string]int{}
+ }
+ m.callErrors[err.Error()]++
+ continue
+ }
+ m.calledAt = time.Now()
+ m.provider = p
+ return nil
+ }
+ if time.Now().After(deadline) {
+ m.fail(stageCall, fmt.Sprintf("%s after %d attempt(s) within %s; last: %s", opts.Service, m.attempts, opts.CallTimeout, lastErr))
+ return errors.New(m.stage)
+ }
+ select {
+ case <-ctx.Done():
+ m.fail(stageCanceled, "while calling "+opts.Service)
+ return ctx.Err()
+ case <-m.exited:
+ m.fail(stageExited, fmt.Sprintf("%v (see %s)", m.exitErr, m.logPath))
+ return errors.New(m.stage)
+ case <-time.After(callRetryInterval):
+ }
+ }
+}
+
+func unixClient(socket string) *http.Client {
+ return &http.Client{Transport: &http.Transport{
+ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+ return (&net.Dialer{}).DialContext(ctx, "unix", socket)
+ },
+ }}
+}
+
+func discover(ctx context.Context, client *http.Client, service string) ([]string, error) {
+ ctx, cancel := context.WithTimeout(ctx, requestTimeout)
+ defer cancel()
+ url := "http://localhost/sam/service/discover?type=mcp&name=" + service + "&timeout=20s"
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("discover: %w", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyProbe))
+ if err != nil {
+ return nil, fmt.Errorf("discover: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("discover: HTTP %d: %s", resp.StatusCode, head(body))
+ }
+ var providers []*api.DiscoveredProvider
+ if err := json.Unmarshal(body, &providers); err != nil {
+ return nil, fmt.Errorf("discover: %w", err)
+ }
+ peers := make([]string, 0, len(providers))
+ for _, p := range providers {
+ if p.GetPeerId() != "" {
+ peers = append(peers, p.GetPeerId())
+ }
+ }
+ return peers, nil
+}
+
+// initialize opens an MCP session with one provider through the mesh. The
+// answer is JSON or an SSE frame; either names the server, and the body is
+// read only as far as that name. A JSON-RPC error arrives as HTTP 200 and is
+// reported by its message.
+func initialize(ctx context.Context, client *http.Client, peer, service string) error {
+ ctx, cancel := context.WithTimeout(ctx, requestTimeout)
+ defer cancel()
+ url := "http://localhost/sam/" + peer + "/mcp/" + service
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(mcpInitialize))
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Accept", "application/json, text/event-stream")
+ resp, err := client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = resp.Body.Close() }()
+
+ var body []byte
+ chunk := make([]byte, 4096)
+ for len(body) < maxBodyProbe {
+ n, err := resp.Body.Read(chunk)
+ body = append(body, chunk[:n]...)
+ if resp.StatusCode == http.StatusOK && bytes.Contains(body, []byte(`"serverInfo"`)) {
+ return nil
+ }
+ if err != nil {
+ break
+ }
+ }
+ if msg := rpcErrorMessage(body); msg != "" {
+ return fmt.Errorf("HTTP %d, JSON-RPC error: %s", resp.StatusCode, msg)
+ }
+ return fmt.Errorf("HTTP %d without serverInfo: %s", resp.StatusCode, head(body))
+}
+
+// rpcErrorMessage extracts the message of a JSON-RPC error response, or the
+// empty string when body is not one.
+func rpcErrorMessage(body []byte) string {
+ var rpc struct {
+ Error *struct {
+ Message string `json:"message"`
+ } `json:"error"`
+ }
+ if json.Unmarshal(bytes.TrimSpace(body), &rpc) != nil || rpc.Error == nil {
+ return ""
+ }
+ return rpc.Error.Message
+}
+
+func head(body []byte) string {
+ s := strings.TrimSpace(string(body))
+ if len(s) > 120 {
+ s = s[:120] + "..."
+ }
+ return s
+}
+
+func summariseJoin(opts JoinOptions, members []*member, elapsed time.Duration) *JoinReport {
+ report := &JoinReport{
+ Count: opts.Count,
+ Ramp: opts.Ramp,
+ Service: opts.Service,
+ Elapsed: elapsed.Seconds(),
+ Errors: map[string]int{},
+ CallErrors: map[string]int{},
+ Members: make([]MemberResult, 0, len(members)),
+ }
+ var ready, call, journey []time.Duration
+ for _, m := range members {
+ report.Members = append(report.Members, m.result())
+ report.CallAttempts += m.attempts
+ for msg, n := range m.callErrors {
+ report.CallErrors[msg] += n
+ }
+ if !m.readyAt.IsZero() {
+ report.Ready++
+ ready = append(ready, m.readyAt.Sub(m.started))
+ }
+ if !m.calledAt.IsZero() {
+ call = append(call, m.calledAt.Sub(m.readyAt))
+ journey = append(journey, m.calledAt.Sub(m.started))
+ }
+ switch {
+ case m.stage != "":
+ report.Failed++
+ report.Errors[m.stage]++
+ default:
+ report.Completed++
+ }
+ }
+ report.ReadyMs = summarise(ready)
+ report.FirstCallMs = summarise(call)
+ report.JourneyMs = summarise(journey)
+ return report
+}
+
+// hold samples every member's readiness until the hold elapses or the
+// context is cancelled, and turns the transitions into outage windows.
+func hold(ctx context.Context, opts JoinOptions, members []*member) *HoldReport {
+ type state struct {
+ ready bool
+ downSince time.Time
+ flapped bool
+ }
+ states := make([]state, len(members))
+ report := &HoldReport{}
+ var outages []time.Duration
+
+ client := &http.Client{Timeout: 2 * time.Second}
+ start := time.Now()
+ end := start.Add(opts.Hold)
+ for {
+ now := time.Now()
+ ready := sampleReady(ctx, client, members)
+ count := 0
+ for i, m := range members {
+ if ready[i] {
+ count++
+ }
+ s := &states[i]
+ switch {
+ case report.Samples == 0:
+ s.ready = ready[i]
+ if !ready[i] {
+ s.downSince = now
+ }
+ case s.ready && !ready[i]:
+ s.ready = false
+ s.downSince = now
+ s.flapped = true
+ report.Flaps++
+ opts.logf("member %04d lost its router", m.index)
+ case !s.ready && ready[i]:
+ s.ready = true
+ outages = append(outages, now.Sub(s.downSince))
+ opts.logf("member %04d back after %.1fs", m.index, now.Sub(s.downSince).Seconds())
+ }
+ }
+ report.Series = append(report.Series, ReadySample{At: now, Ready: count})
+ if report.Samples == 0 {
+ report.Resident = count
+ report.MinReady = count
+ } else if count < report.MinReady {
+ report.MinReady = count
+ }
+ report.Samples++
+
+ if now.After(end) || ctx.Err() != nil {
+ break
+ }
+ select {
+ case <-ctx.Done():
+ case <-time.After(opts.SampleInterval):
+ }
+ if ctx.Err() != nil {
+ break
+ }
+ }
+
+ report.Seconds = time.Since(start).Seconds()
+ report.OutageMs = summarise(outages)
+ for i, m := range members {
+ if states[i].flapped {
+ report.Flapped++
+ }
+ if !states[i].ready {
+ report.NotReadyAtEnd++
+ }
+ if m.cmd != nil && m.hasExited() {
+ report.Exited++
+ }
+ }
+ return report
+}
+
+// sampleReady reads every member's /readyz, a bounded number at a time.
+func sampleReady(ctx context.Context, client *http.Client, members []*member) []bool {
+ ready := make([]bool, len(members))
+ sem := make(chan struct{}, sampleConcurrency)
+ var wg sync.WaitGroup
+ for i, m := range members {
+ if m.cmd == nil || m.hasExited() {
+ continue
+ }
+ wg.Add(1)
+ sem <- struct{}{}
+ go func() {
+ defer wg.Done()
+ defer func() { <-sem }()
+ ready[i] = isReady(ctx, client, m.readyz)
+ }()
+ }
+ wg.Wait()
+ return ready
+}
+
+// teardown stops every member: a SIGTERM, then a SIGKILL for whatever is
+// still running when the grace period ends.
+func teardown(members []*member) {
+ for _, m := range members {
+ if m.cmd != nil && !m.hasExited() {
+ _ = m.cmd.Process.Signal(syscall.SIGTERM)
+ }
+ }
+ deadline := time.Now().Add(teardownGrace)
+ for _, m := range members {
+ if m.cmd == nil {
+ continue
+ }
+ select {
+ case <-m.exited:
+ case <-time.After(time.Until(deadline)):
+ _ = m.cmd.Process.Kill()
+ <-m.exited
+ }
+ _ = m.logFile.Close()
+ }
+}
+
+func millis(d time.Duration) float64 {
+ return float64(d.Nanoseconds()) / 1e6
+}
diff --git a/internal/bench/join_test.go b/internal/bench/join_test.go
new file mode 100644
index 00000000..20804973
--- /dev/null
+++ b/internal/bench/join_test.go
@@ -0,0 +1,379 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bench
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net"
+ "net/http"
+ "os"
+ "os/signal"
+ "path/filepath"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+)
+
+// The members RunJoin starts are whatever binary it is pointed at. Here that
+// is this test binary again, re-entered through TestMain as a node that
+// answers /readyz and the two sidecar routes the journey uses, with its
+// behaviour set through the environment the children inherit.
+
+func TestMain(m *testing.M) {
+ if os.Getenv("SAM_BENCH_FAKE_NODE") == "1" {
+ os.Exit(fakeNode(os.Args[1:]))
+ }
+ os.Exit(m.Run())
+}
+
+func fakeNode(args []string) int {
+ var metricsAddr, socket, tokenPath string
+ for i := 0; i < len(args)-1; i++ {
+ switch args[i] {
+ case "--metrics-addr":
+ metricsAddr = args[i+1]
+ case "--socket-path":
+ socket = args[i+1]
+ case "--bootstrap-token-path":
+ tokenPath = args[i+1]
+ }
+ }
+ if os.Getenv("FAKE_NODE_EXIT") != "" {
+ fmt.Fprintln(os.Stderr, "fake node: refusing to start")
+ return 3
+ }
+ if tokenPath == "" {
+ fmt.Fprintln(os.Stderr, "fake node: no --bootstrap-token-path")
+ return 2
+ }
+
+ readyAfter, _ := time.ParseDuration(os.Getenv("FAKE_NODE_READY_AFTER"))
+ flapAt, _ := time.ParseDuration(os.Getenv("FAKE_NODE_FLAP_AT"))
+ flapFor, _ := time.ParseDuration(os.Getenv("FAKE_NODE_FLAP_FOR"))
+ providers := strings.Split(os.Getenv("FAKE_NODE_PROVIDERS"), ",")
+ start := time.Now()
+ ready := func() bool {
+ elapsed := time.Since(start)
+ if elapsed < readyAfter {
+ return false
+ }
+ return flapAt == 0 || elapsed < flapAt || elapsed >= flapAt+flapFor
+ }
+
+ metrics := http.NewServeMux()
+ metrics.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
+ if !ready() {
+ http.Error(w, "not ready", http.StatusServiceUnavailable)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ })
+ sidecar := http.NewServeMux()
+ sidecar.HandleFunc("/sam/service/discover", func(w http.ResponseWriter, _ *http.Request) {
+ var out []map[string]string
+ for _, p := range providers {
+ if p != "" {
+ out = append(out, map[string]string{"peer_id": p})
+ }
+ }
+ _ = json.NewEncoder(w).Encode(out)
+ })
+ sidecar.HandleFunc("/sam/", func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasPrefix(r.URL.Path, "/sam/dead/") {
+ http.Error(w, "no route to peer", http.StatusBadGateway)
+ return
+ }
+ _, _ = fmt.Fprint(w, `{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-06-18","serverInfo":{"name":"fake","version":"0"}}}`)
+ })
+
+ tcp, err := net.Listen("tcp", metricsAddr)
+ if err != nil {
+ fmt.Fprintln(os.Stderr, "fake node:", err)
+ return 1
+ }
+ unix, err := net.Listen("unix", socket)
+ if err != nil {
+ fmt.Fprintln(os.Stderr, "fake node:", err)
+ return 1
+ }
+ go func() { _ = http.Serve(tcp, metrics) }() //nolint:gosec // a test double
+ go func() { _ = http.Serve(unix, sidecar) }() //nolint:gosec // a test double
+ stop := make(chan os.Signal, 1)
+ signal.Notify(stop, syscall.SIGTERM, os.Interrupt)
+ <-stop
+ return 0
+}
+
+func fakeFleet(t *testing.T, count int, env map[string]string) JoinOptions {
+ t.Helper()
+ t.Setenv("SAM_BENCH_FAKE_NODE", "1")
+ if env == nil {
+ env = map[string]string{}
+ }
+ if _, ok := env["FAKE_NODE_PROVIDERS"]; !ok {
+ env["FAKE_NODE_PROVIDERS"] = "alive"
+ }
+ for _, k := range []string{"FAKE_NODE_EXIT", "FAKE_NODE_READY_AFTER", "FAKE_NODE_FLAP_AT", "FAKE_NODE_FLAP_FOR", "FAKE_NODE_PROVIDERS"} {
+ t.Setenv(k, env[k])
+ }
+ token := filepath.Join(t.TempDir(), "token")
+ if err := os.WriteFile(token, []byte("sam-bt-test\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ // t.TempDir carries the test's name, which for a subtest is long enough
+ // to push a member's socket past the Unix path limit.
+ dir, err := os.MkdirTemp("", "sam-join-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.RemoveAll(dir) })
+ return JoinOptions{
+ NodeBin: os.Args[0],
+ Count: count,
+ Dir: filepath.Join(dir, "fleet"),
+ ControlPlane: "http://127.0.0.1:1",
+ BootstrapTokenPath: token,
+ Service: "everything",
+ MetricsBasePort: freePortBase(t, count),
+ }
+}
+
+// freePortBase finds count consecutive free loopback ports.
+func freePortBase(t *testing.T, count int) int {
+ t.Helper()
+ for attempt := 0; attempt < 20; attempt++ {
+ l, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ base := l.Addr().(*net.TCPAddr).Port
+ _ = l.Close()
+ free := true
+ for i := 1; i < count; i++ {
+ probe, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i))
+ if err != nil {
+ free = false
+ break
+ }
+ _ = probe.Close()
+ }
+ if free {
+ return base
+ }
+ }
+ t.Fatal("no run of free ports")
+ return 0
+}
+
+func TestRunJoinTimesEachStageAndRetriesStaleProviders(t *testing.T) {
+ // A provider record that names a peer which does not answer is the
+ // normal state of a mesh just after a rollout. The journey has to get
+ // past it and the report has to say that it happened.
+ opts := fakeFleet(t, 3, map[string]string{
+ "FAKE_NODE_READY_AFTER": "200ms",
+ "FAKE_NODE_PROVIDERS": "dead,alive",
+ })
+
+ report, err := RunJoin(context.Background(), opts)
+ if err != nil {
+ t.Fatalf("RunJoin: %v", err)
+ }
+ if report.Ready != 3 || report.Completed != 3 || report.Failed != 0 {
+ t.Fatalf("ready/completed/failed = %d/%d/%d, want 3/3/0 (errors %v)", report.Ready, report.Completed, report.Failed, report.Errors)
+ }
+ if report.ReadyMs.Count != 3 || report.ReadyMs.Min < 200 {
+ t.Errorf("ready_ms = %+v, want 3 samples none under the 200ms the node took", report.ReadyMs)
+ }
+ if report.JourneyMs.Count != 3 || report.FirstCallMs.Count != 3 {
+ t.Errorf("journey/first-call counts = %d/%d, want 3/3", report.JourneyMs.Count, report.FirstCallMs.Count)
+ }
+ if report.CallAttempts != 6 {
+ t.Errorf("call_attempts = %d, want 6: the dead provider tried once per member before the live one", report.CallAttempts)
+ }
+ if len(report.CallErrors) != 1 {
+ t.Errorf("call_errors = %v, want the dead provider's refusal once per member", report.CallErrors)
+ }
+ for msg, n := range report.CallErrors {
+ if n != 3 || !strings.Contains(msg, "HTTP 502") {
+ t.Errorf("call_errors = %v, want 3 x HTTP 502", report.CallErrors)
+ }
+ }
+ for _, m := range report.Members {
+ if m.Provider != "alive" || m.Attempts != 2 || m.Error != "" {
+ t.Errorf("member %d = %+v, want provider alive after 2 attempts", m.Index, m)
+ }
+ if _, err := os.Stat(filepath.Join(m.Dir, "node.log")); err != nil {
+ t.Errorf("member %d kept no log: %v", m.Index, err)
+ }
+ }
+ if report.Hold != nil {
+ t.Errorf("hold reported without --hold")
+ }
+}
+
+func TestRunJoinReportsMembersLeftBehindByStage(t *testing.T) {
+ // A fleet that mostly failed must not read as a fast one: a member that
+ // never got there is counted, by the stage it stopped at, and never
+ // contributes a latency.
+ t.Run("process exits", func(t *testing.T) {
+ opts := fakeFleet(t, 2, map[string]string{"FAKE_NODE_EXIT": "1"})
+ report, err := RunJoin(context.Background(), opts)
+ if err != nil {
+ t.Fatalf("RunJoin: %v", err)
+ }
+ if report.Failed != 2 || report.Errors[stageExited] != 2 {
+ t.Fatalf("failed = %d, errors = %v, want 2 under %q", report.Failed, report.Errors, stageExited)
+ }
+ if report.ReadyMs.Count != 0 {
+ t.Errorf("a member that exited contributed a readiness latency")
+ }
+ if !strings.Contains(report.Members[0].Error, "exit status 3") {
+ t.Errorf("member error = %q, want the exit status", report.Members[0].Error)
+ }
+ })
+ t.Run("never ready", func(t *testing.T) {
+ opts := fakeFleet(t, 2, map[string]string{"FAKE_NODE_READY_AFTER": "1h"})
+ opts.ReadyTimeout = 300 * time.Millisecond
+ report, err := RunJoin(context.Background(), opts)
+ if err != nil {
+ t.Fatalf("RunJoin: %v", err)
+ }
+ if report.Failed != 2 || report.Errors[stageReady] != 2 {
+ t.Fatalf("failed = %d, errors = %v, want 2 under %q", report.Failed, report.Errors, stageReady)
+ }
+ })
+ t.Run("no provider answers", func(t *testing.T) {
+ opts := fakeFleet(t, 1, map[string]string{"FAKE_NODE_PROVIDERS": "dead"})
+ opts.CallTimeout = 100 * time.Millisecond
+ report, err := RunJoin(context.Background(), opts)
+ if err != nil {
+ t.Fatalf("RunJoin: %v", err)
+ }
+ if report.Ready != 1 || report.Completed != 0 || report.Errors[stageCall] != 1 {
+ t.Fatalf("ready/completed = %d/%d, errors = %v, want 1/0 under %q", report.Ready, report.Completed, report.Errors, stageCall)
+ }
+ if report.ReadyMs.Count != 1 || report.JourneyMs.Count != 0 {
+ t.Errorf("ready/journey counts = %d/%d: readiness was reached, the journey was not", report.ReadyMs.Count, report.JourneyMs.Count)
+ }
+ })
+}
+
+func TestRunJoinHoldTurnsFlapsIntoOutageWindows(t *testing.T) {
+ // What a resident fleet is for: a router that goes away for a while
+ // shows up as one outage per member with a length, and a fleet that is
+ // whole again at the end says so.
+ opts := fakeFleet(t, 2, map[string]string{
+ "FAKE_NODE_FLAP_AT": "1200ms",
+ "FAKE_NODE_FLAP_FOR": "400ms",
+ })
+ opts.Hold = 2 * time.Second
+ opts.SampleInterval = 100 * time.Millisecond
+ var residentCompleted int
+ var residentHadHold bool
+ opts.OnResident = func(r *JoinReport) { residentCompleted, residentHadHold = r.Completed, r.Hold != nil }
+
+ report, err := RunJoin(context.Background(), opts)
+ if err != nil {
+ t.Fatalf("RunJoin: %v", err)
+ }
+ if report.Completed != 2 {
+ t.Fatalf("completed = %d, want 2 (errors %v)", report.Completed, report.Errors)
+ }
+ if residentCompleted != 2 || residentHadHold {
+ t.Errorf("OnResident saw completed=%d hold=%v, want the journey report before the hold", residentCompleted, residentHadHold)
+ }
+ h := report.Hold
+ if h == nil {
+ t.Fatal("no hold report")
+ }
+ if h.Resident != 2 || h.MinReady != 0 {
+ t.Errorf("resident/min_ready = %d/%d, want 2/0", h.Resident, h.MinReady)
+ }
+ if h.Flaps != 2 || h.Flapped != 2 || h.OutageMs.Count != 2 {
+ t.Errorf("flaps/flapped/outages = %d/%d/%d, want 2/2/2", h.Flaps, h.Flapped, h.OutageMs.Count)
+ }
+ if h.OutageMs.Min < 300 || h.OutageMs.Max > 900 {
+ t.Errorf("outage_ms = %+v, want windows around the 400ms the node was away", h.OutageMs)
+ }
+ if h.NotReadyAtEnd != 0 || h.Exited != 0 {
+ t.Errorf("not_ready_at_end/exited = %d/%d, want 0/0", h.NotReadyAtEnd, h.Exited)
+ }
+ if h.Samples < 10 || len(h.Series) != h.Samples {
+ t.Errorf("samples = %d, series = %d, want at least 10 and equal", h.Samples, len(h.Series))
+ }
+}
+
+func TestRunJoinCancelEndsTheHoldWithAReport(t *testing.T) {
+ // Ctrl-C during a long hold is how most holds end. The report is the
+ // point of the run and must survive it.
+ opts := fakeFleet(t, 1, nil)
+ opts.Hold = time.Hour
+ opts.SampleInterval = 50 * time.Millisecond
+
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ time.Sleep(1500 * time.Millisecond)
+ cancel()
+ }()
+ done := make(chan struct{})
+ var report *JoinReport
+ var err error
+ go func() {
+ report, err = RunJoin(ctx, opts)
+ close(done)
+ }()
+ select {
+ case <-done:
+ case <-time.After(15 * time.Second):
+ t.Fatal("RunJoin did not return after cancel")
+ }
+ if err != nil {
+ t.Fatalf("RunJoin: %v", err)
+ }
+ if report.Hold == nil || report.Hold.Samples == 0 {
+ t.Fatalf("hold = %+v, want samples from before the cancel", report.Hold)
+ }
+ if report.Hold.Seconds > 60 {
+ t.Errorf("hold ran %.0fs, want it ended by the cancel", report.Hold.Seconds)
+ }
+}
+
+func TestRunJoinRefusesADirectoryThatHoldsState(t *testing.T) {
+ opts := fakeFleet(t, 1, nil)
+ if err := os.MkdirAll(filepath.Join(opts.Dir, "member-0000"), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := RunJoin(context.Background(), opts); err == nil || !strings.Contains(err.Error(), "not empty") {
+ t.Fatalf("err = %v, want a refusal to reuse member state", err)
+ }
+}
+
+func TestRunJoinRampSpacesTheStarts(t *testing.T) {
+ opts := fakeFleet(t, 3, nil)
+ opts.Ramp = 5
+ report, err := RunJoin(context.Background(), opts)
+ if err != nil {
+ t.Fatalf("RunJoin: %v", err)
+ }
+ if report.Elapsed < 0.4 {
+ t.Errorf("elapsed = %.2fs, want at least the 0.4s two gaps at 5/s take", report.Elapsed)
+ }
+ if report.Completed != 3 {
+ t.Errorf("completed = %d, want 3 (errors %v)", report.Completed, report.Errors)
+ }
+}