Skip to content

Agent Mesh

node, bench: /readyz tells the truth on every listener; a member binds no loopback port

Antonio Ojea6e308ff3c91d
diff --git a/cmd/agentmesh-bench/main.go b/cmd/agentmesh-bench/main.go
index 3968e77b..01b69d1a 100644
--- a/cmd/agentmesh-bench/main.go
+++ b/cmd/agentmesh-bench/main.go
@@ -282,7 +282,6 @@ func newJoinCmd() *cobra.Command {
flags.DurationVar(&opts.CallTimeout, "call-timeout", 120*time.Second, "Bound on one member's discovery and first call, from readiness")
flags.DurationVar(&opts.Hold, "hold", 0, "Keep the fleet resident this long after the journeys end, sampling readiness; 0 tears it down at once")
flags.DurationVar(&opts.SampleInterval, "sample-interval", 10*time.Second, "How often each resident member's readiness is read during --hold")
- flags.IntVar(&opts.MetricsBasePort, "metrics-base-port", 20000, "Member i serves /readyz on 127.0.0.1:(port+i)")
flags.StringArrayVar(&opts.NodeArgs, "node-arg", nil, "Extra argument for every agentmesh-node, repeatable (e.g. --node-arg=--insecure-control-plane)")
flags.StringVar(&marker, "resident-marker", "", "File to create once the fleet is resident and the hold has begun")
flags.StringArrayVar(&scrape, "scrape", nil, "Metrics endpoint to record before and after the run, repeatable")
diff --git a/internal/bench/join.go b/internal/bench/join.go
index 91eceb64..ac7904eb 100644
--- a/internal/bench/join.go
+++ b/internal/bench/join.go
@@ -93,10 +93,6 @@ type JoinOptions struct {
// SampleInterval is how often each resident member is read during Hold.
SampleInterval time.Duration
- // MetricsBasePort is the first of Count consecutive loopback ports;
- // member i serves /readyz on MetricsBasePort+i.
- MetricsBasePort int
-
// NodeArgs are appended to every member's command line as given.
NodeArgs []string
@@ -208,6 +204,7 @@ const (
// and one that takes a minute is not asked a thousand times.
readyPollMin = 50 * time.Millisecond
readyPollMax = 500 * time.Millisecond
+ readyTimeout = 2 * time.Second
callRetryInterval = 5 * time.Second
requestTimeout = 30 * time.Second
teardownGrace = 10 * time.Second
@@ -222,17 +219,19 @@ const (
const mcpInitialize = `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"agentmesh-bench","version":"0"}}}`
type member struct {
- index int
- dir string
- socket string
- metricsAddr string
- readyz string
- logPath string
- cmd *exec.Cmd
- logFile *os.File
- started time.Time
- exited chan struct{}
- exitErr error
+ index int
+ dir string
+ socket string
+ // client speaks to the member over its Unix socket: readiness, discovery
+ // and the first call. A member binds no loopback port, so a fleet of any
+ // size cannot collide with the kernel's ephemeral range.
+ client *http.Client
+ logPath string
+ cmd *exec.Cmd
+ logFile *os.File
+ started time.Time
+ exited chan struct{}
+ exitErr error
readyAt time.Time
calledAt time.Time
@@ -364,9 +363,6 @@ func (o *JoinOptions) setDefaults() {
if o.SampleInterval <= 0 {
o.SampleInterval = 10 * time.Second
}
- if o.MetricsBasePort <= 0 {
- o.MetricsBasePort = 20000
- }
}
func (o *JoinOptions) logf(format string, args ...any) {
@@ -378,14 +374,13 @@ func (o *JoinOptions) logf(format string, args ...any) {
func newMember(opts JoinOptions, i int) *member {
dir := filepath.Join(opts.Dir, fmt.Sprintf("member-%04d", i))
- metricsAddr := fmt.Sprintf("127.0.0.1:%d", opts.MetricsBasePort+i)
+ socket := filepath.Join(dir, "node.sock")
return &member{
- index: i,
- dir: dir,
- socket: filepath.Join(dir, "node.sock"),
- metricsAddr: metricsAddr,
- readyz: "http://" + metricsAddr + "/readyz",
- logPath: filepath.Join(dir, "node.log"),
+ index: i,
+ dir: dir,
+ socket: socket,
+ client: unixClient(socket),
+ logPath: filepath.Join(dir, "node.log"),
}
}
@@ -407,7 +402,6 @@ func (m *member) start(opts JoinOptions) error {
"--data-dir", m.dir,
"--socket-path", m.socket,
"--bind-addr=",
- "--metrics-addr", m.metricsAddr,
"--listen", "/ip4/0.0.0.0/tcp/0",
"--listen", "/ip4/0.0.0.0/udp/0/quic-v1",
}
@@ -456,11 +450,10 @@ func journey(ctx context.Context, opts JoinOptions, m *member) {
// waitReady polls the member's /readyz, which answers 200 only once the node
// holds an authenticated connection to a router.
func (m *member) waitReady(ctx context.Context, timeout time.Duration) error {
- client := &http.Client{Timeout: 2 * time.Second}
deadline := time.Now().Add(timeout)
interval := readyPollMin
for {
- if isReady(ctx, client, m.readyz) {
+ if m.isReady(ctx) {
m.readyAt = time.Now()
return nil
}
@@ -481,12 +474,14 @@ func (m *member) waitReady(ctx context.Context, timeout time.Duration) error {
}
}
-func isReady(ctx context.Context, client *http.Client, url string) bool {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+func (m *member) isReady(ctx context.Context) bool {
+ ctx, cancel := context.WithTimeout(ctx, readyTimeout)
+ defer cancel()
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost/readyz", nil)
if err != nil {
return false
}
- resp, err := client.Do(req)
+ resp, err := m.client.Do(req)
if err != nil {
return false
}
@@ -500,7 +495,7 @@ func isReady(ctx context.Context, client *http.Client, url string) bool {
// deadline: a provider record can name a peer a rollout just replaced, and a
// member that joined seconds ago has a thin routing table.
func (m *member) firstCall(ctx context.Context, opts JoinOptions) error {
- client := unixClient(m.socket)
+ client := m.client
defer client.CloseIdleConnections()
deadline := m.readyAt.Add(opts.CallTimeout)
lastErr := "none discovered"
@@ -691,12 +686,11 @@ func hold(ctx context.Context, opts JoinOptions, members []*member) *HoldReport
report := &HoldReport{}
var outages []time.Duration
- client := &http.Client{Timeout: 2 * time.Second}
start := time.Now()
end := start.Add(opts.Hold)
for {
now := time.Now()
- ready := sampleReady(ctx, client, members)
+ ready := sampleReady(ctx, members)
count := 0
for i, m := range members {
if ready[i] {
@@ -759,7 +753,7 @@ func hold(ctx context.Context, opts JoinOptions, members []*member) *HoldReport
}
// sampleReady reads every member's /readyz, a bounded number at a time.
-func sampleReady(ctx context.Context, client *http.Client, members []*member) []bool {
+func sampleReady(ctx context.Context, members []*member) []bool {
ready := make([]bool, len(members))
sem := make(chan struct{}, sampleConcurrency)
var wg sync.WaitGroup
@@ -772,7 +766,7 @@ func sampleReady(ctx context.Context, client *http.Client, members []*member) []
go func() {
defer wg.Done()
defer func() { <-sem }()
- ready[i] = isReady(ctx, client, m.readyz)
+ ready[i] = m.isReady(ctx)
}()
}
wg.Wait()
diff --git a/internal/bench/join_test.go b/internal/bench/join_test.go
index 1ee3306a..1191efb1 100644
--- a/internal/bench/join_test.go
+++ b/internal/bench/join_test.go
@@ -31,8 +31,11 @@ import (
// The members RunJoin starts are whatever binary it is pointed at. Here that
// is this test binary again, re-entered through TestMain as a node that
-// answers /readyz and the two sidecar routes the journey uses, with its
-// behaviour set through the environment the children inherit.
+// answers /readyz and the two sidecar routes the journey uses on its Unix
+// socket, with its behaviour set through the environment the children
+// inherit. Like the real node it binds no loopback port unless told to, and
+// it refuses to be told: a fleet of a thousand on one host collided with the
+// kernel's ephemeral range when every member had one.
func TestMain(m *testing.M) {
if os.Getenv("AGENTMESH_BENCH_FAKE_NODE") == "1" {
@@ -42,11 +45,14 @@ func TestMain(m *testing.M) {
}
func fakeNode(args []string) int {
- var metricsAddr, socket, tokenPath string
+ var socket, tokenPath string
for i := 0; i < len(args)-1; i++ {
switch args[i] {
- case "--metrics-addr":
- metricsAddr = args[i+1]
+ case "--metrics-addr", "--bind-addr":
+ if args[i+1] != "" {
+ fmt.Fprintln(os.Stderr, "fake node: a member must not bind a loopback port:", args[i], args[i+1])
+ return 4
+ }
case "--socket-path":
socket = args[i+1]
case "--bootstrap-token-path":
@@ -78,15 +84,14 @@ func fakeNode(args []string) int {
return true
}
- metrics := http.NewServeMux()
- metrics.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
+ sidecar := http.NewServeMux()
+ sidecar.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
if !ready() {
http.Error(w, "not ready", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
})
- sidecar := http.NewServeMux()
sidecar.HandleFunc("/mesh/service/discover", func(w http.ResponseWriter, _ *http.Request) {
var out []map[string]string
for _, p := range providers {
@@ -104,17 +109,11 @@ func fakeNode(args []string) int {
_, _ = fmt.Fprint(w, `{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-06-18","serverInfo":{"name":"fake","version":"0"}}}`)
})
- tcp, err := net.Listen("tcp", metricsAddr)
- if err != nil {
- fmt.Fprintln(os.Stderr, "fake node:", err)
- return 1
- }
unix, err := net.Listen("unix", socket)
if err != nil {
fmt.Fprintln(os.Stderr, "fake node:", err)
return 1
}
- go func() { _ = http.Serve(tcp, metrics) }() //nolint:gosec // a test double
go func() { _ = http.Serve(unix, sidecar) }() //nolint:gosec // a test double
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGTERM, os.Interrupt)
@@ -152,37 +151,9 @@ func fakeFleet(t *testing.T, count int, env map[string]string) JoinOptions {
ControlPlane: "http://127.0.0.1:1",
BootstrapTokenPath: token,
Service: "everything",
- MetricsBasePort: freePortBase(t, count),
}
}
-// freePortBase finds count consecutive free loopback ports.
-func freePortBase(t *testing.T, count int) int {
- t.Helper()
- for attempt := 0; attempt < 20; attempt++ {
- l, err := net.Listen("tcp", "127.0.0.1:0")
- if err != nil {
- t.Fatal(err)
- }
- base := l.Addr().(*net.TCPAddr).Port
- _ = l.Close()
- free := true
- for i := 1; i < count; i++ {
- probe, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i))
- if err != nil {
- free = false
- break
- }
- _ = probe.Close()
- }
- if free {
- return base
- }
- }
- t.Fatal("no run of free ports")
- return 0
-}
-
func TestRunJoinTimesEachStageAndRetriesStaleProviders(t *testing.T) {
// A provider record that names a peer which does not answer is the
// normal state of a mesh just after a rollout. The journey has to get
diff --git a/internal/node/sidecar.go b/internal/node/sidecar.go
index 79f3fd7b..00ba4b40 100644
--- a/internal/node/sidecar.go
+++ b/internal/node/sidecar.go
@@ -47,7 +47,7 @@ func StartSidecarServer(node *AgentMeshNode, addr, socketPath, token, certFile,
// Public endpoints
mux.HandleFunc("/healthz", handleHealthz)
- mux.HandleFunc("/readyz", handleReadyz)
+ mux.HandleFunc("/readyz", handleReadyzFor(node))
mux.HandleFunc("/.well-known/oauth-protected-resource", func(w http.ResponseWriter, r *http.Request) {
handleOAuthProtectedResource(node, w, r)
})
@@ -315,7 +315,7 @@ func StartUnauthSidecarServer(controlPlaneURL, addr, socketPath, certFile, keyFi
// Public endpoints
mux.HandleFunc("/healthz", handleHealthz)
- mux.HandleFunc("/readyz", handleReadyz)
+ mux.HandleFunc("/readyz", handleReadyzFor(nil))
// Mount Unauthenticated MCP handler
mcpHandler := NewUnauthenticatedMCPHandler(controlPlaneURL)
@@ -383,10 +383,20 @@ func handleHealthz(w http.ResponseWriter, r *http.Request) {
}
}
-func handleReadyz(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(http.StatusOK)
- if _, err := w.Write([]byte("OK")); err != nil {
- logger.Errorf("Failed to write response: %v", err)
+// handleReadyzFor answers 200 only once the node holds an authenticated
+// connection to a router, the same answer the metrics listener gives, so a
+// probe or a harness reading either listener learns the same thing. A node
+// without an identity yet (nil) is not ready.
+func handleReadyzFor(node *AgentMeshNode) http.HandlerFunc {
+ return func(w http.ResponseWriter, _ *http.Request) {
+ if node == nil || node.debugReady() != nil || !node.IsConnected() {
+ http.Error(w, "not connected to the mesh", http.StatusServiceUnavailable)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ if _, err := w.Write([]byte("OK")); err != nil {
+ logger.Errorf("Failed to write response: %v", err)
+ }
}
}
diff --git a/internal/node/sidecar_auth_test.go b/internal/node/sidecar_auth_test.go
index b3346524..34405b53 100644
--- a/internal/node/sidecar_auth_test.go
+++ b/internal/node/sidecar_auth_test.go
@@ -19,6 +19,9 @@ import (
"path/filepath"
"testing"
"time"
+
+ "github.com/libp2p/go-libp2p"
+ "github.com/libp2p/go-libp2p/core/peer"
)
// TestConstantTimeEqual is a correctness check, not a timing one: a timing
@@ -57,6 +60,80 @@ func TestConstantTimeEqual(t *testing.T) {
}
}
+// TestSidecarReadyzFollowsTheMeshConnection pins that the sidecar's /readyz
+// says what the metrics listener's does: 503 until the node holds an
+// authenticated, connected router, then 200, on the socket and on TCP. A
+// harness that reads readiness over the socket, as agentmesh-bench does so
+// that a member binds no loopback port, learns the same thing a kubelet
+// probing the metrics port would.
+func TestSidecarReadyzFollowsTheMeshConnection(t *testing.T) {
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+ host, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = host.Close() }()
+ node := &AgentMeshNode{
+ Host: host,
+ Store: store,
+ BiscuitTimeout: 500 * time.Millisecond,
+ services: NewServiceRegistry(&fakeDHT{}, 0),
+ }
+ node.Directory = &routerDirectory{host: host, routers: node.attachedRouters, addrs: host.Addrs}
+ socketPath := filepath.Join(t.TempDir(), "agentmesh.sock")
+ srv, err := StartSidecarServer(node, "127.0.0.1:0", socketPath, "test-token", "", "", "")
+ if err != nil {
+ t.Fatalf("failed to start sidecar server: %v", err)
+ }
+ defer func() { _ = srv.Close() }()
+ socket := waitForSocket(t, socketPath)
+ tcp := &http.Client{Timeout: 2 * time.Second}
+
+ readyz := func(t *testing.T, client *http.Client, url string) int {
+ t.Helper()
+ resp, err := client.Get(url)
+ if err != nil {
+ t.Fatalf("GET %s: %v", url, err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ return resp.StatusCode
+ }
+ for name, url := range map[string]string{"socket": "http://localhost/readyz", "tcp": "http://" + node.BoundHTTPAddr + "/readyz"} {
+ client := socket
+ if name == "tcp" {
+ client = tcp
+ }
+ if code := readyz(t, client, url); code != http.StatusServiceUnavailable {
+ t.Errorf("%s /readyz = %d with no authenticated router, want 503", name, code)
+ }
+ }
+
+ routerHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = routerHost.Close() }()
+ if err := host.Connect(t.Context(), peer.AddrInfo{ID: routerHost.ID(), Addrs: routerHost.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+ node.mu.Lock()
+ node.authenticatedRouters = map[peer.ID]bool{routerHost.ID(): true}
+ node.mu.Unlock()
+ for name, url := range map[string]string{"socket": "http://localhost/readyz", "tcp": "http://" + node.BoundHTTPAddr + "/readyz"} {
+ client := socket
+ if name == "tcp" {
+ client = tcp
+ }
+ if code := readyz(t, client, url); code != http.StatusOK {
+ t.Errorf("%s /readyz = %d once a router is authenticated and connected, want 200", name, code)
+ }
+ }
+}
+
// TestMetricsGatedOnTCPButNotOnTheSocket pins where the metrics endpoint sits in
// the sidecar's trust model. Its labels carry peer IDs and per-peer request
// counts, and the TCP listener is reachable by any local process, so it is
diff --git a/internal/node/sidecar_test.go b/internal/node/sidecar_test.go
index 17b55e8a..0f25ea78 100644
--- a/internal/node/sidecar_test.go
+++ b/internal/node/sidecar_test.go
@@ -399,7 +399,8 @@ func TestSidecarServerAuthEnforcement(t *testing.T) {
needsToken bool
}{
{"healthz is public", "GET", "/healthz", http.StatusOK, false},
- {"readyz is public", "GET", "/readyz", http.StatusOK, false},
+ // Public, and honest: this node holds no router, so it is not ready.
+ {"readyz is public", "GET", "/readyz", http.StatusServiceUnavailable, false},
// Liveness says nothing; metrics name peers and count their traffic.
{"metrics is protected", "GET", "/metrics", http.StatusUnauthorized, false},
// Services are declared in configuration only: the former runtime
diff --git a/tests/integration/member_burst_test.go b/tests/integration/member_burst_test.go
index e5940169..d1a590bb 100644
--- a/tests/integration/member_burst_test.go
+++ b/tests/integration/member_burst_test.go
@@ -16,7 +16,6 @@ package integration_test
import (
"context"
- "fmt"
"net"
"net/http/httptest"
"os"
@@ -92,7 +91,6 @@ func TestMemberBurstJoinsConcurrently(t *testing.T) {
Service: "everything",
ReadyTimeout: 30 * time.Second,
CallTimeout: 30 * time.Second,
- MetricsBasePort: reservePortBlock(t, members),
NodeArgs: []string{"--allow-loopback"},
Log: testLogWriter{t},
})
@@ -134,29 +132,6 @@ func TestMemberBurstJoinsConcurrently(t *testing.T) {
members, report.ReadyMs.P50, report.ReadyMs.P95, report.JourneyMs.P50, report.JourneyMs.P95, report.JourneyMs.Max, report.CallAttempts)
}
-// reservePortBlock finds n consecutive free loopback ports in the reserved
-// range and returns the first.
-func reservePortBlock(t *testing.T, n int) int {
- t.Helper()
- for range 50 {
- base := getFreePort(t)
- free := true
- for i := 1; i < n; i++ {
- l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i))
- if err != nil {
- free = false
- break
- }
- _ = l.Close()
- }
- if free {
- return base
- }
- }
- t.Fatalf("no run of %d free ports", n)
- return 0
-}
-
// logTail returns the last part of a log file, for a failure message.
func logTail(path string) string {
data, err := os.ReadFile(path) // #nosec G304 -- a log the test wrote
diff --git a/tests/scale/member-journey.sh b/tests/scale/member-journey.sh
index 2c67d100..ee38e309 100755
--- a/tests/scale/member-journey.sh
+++ b/tests/scale/member-journey.sh
@@ -259,16 +259,15 @@ summarize() {
# a backgrounded function is a subshell, and a signal to it would not reach
# the run inside.
JOIN_CMD=()
-join_cmd() { # phase count base-port extra...
- local phase="$1" count="$2" base="$3" a
- shift 3
+join_cmd() { # phase count extra...
+ local phase="$1" count="$2" a
+ shift 2
JOIN_CMD=("$AGENTMESH_BENCH" join
--node-bin "$AGENTMESH_NODE"
--control-plane "$CONTROL_PLANE"
--bootstrap-token-path "$BOOTSTRAP_TOKEN_PATH"
--service "$SERVICE"
--count "$count"
- --metrics-base-port "$base"
--dir "$OUT/$phase"
--label "phase=$phase" --label "count=$count" --label "burst=$COUNT" --label "host=$(hostname)"
--out "$OUT/$phase.json")
@@ -288,7 +287,7 @@ rm -f "$OUT/resident"
snapshot before
event burst-start
log "burst: ${COUNT} members at ${RAMP}/s, then holding ${HOLD}"
-join_cmd burst "$COUNT" 20000 --ramp "$RAMP" --hold "$HOLD" --resident-marker "$OUT/resident"
+join_cmd burst "$COUNT" --ramp "$RAMP" --hold "$HOLD" --resident-marker "$OUT/resident"
"${JOIN_CMD[@]}" 2> "$OUT/burst.log" &
JOIN_PID=$!
while [[ ! -f "$OUT/resident" ]]; do
@@ -309,7 +308,7 @@ PY
if (( LATE > 0 )); then
event late-start
log "late: ${LATE} members join the populated mesh"
- join_cmd late "$LATE" 30000
+ join_cmd late "$LATE"
"${JOIN_CMD[@]}" 2> "$OUT/late.log" || log "late phase did not run cleanly; see $OUT/late.log"
event late-end
fi
@@ -326,7 +325,7 @@ if (( ROLLOUT )); then
# timer; a long bound here measures that wait instead of hiding it.
event late-after-rollout-start
log "late: ${LATE} members join behind the rollout"
- join_cmd late-after-rollout "$LATE" 31000 --ready-timeout 600s
+ join_cmd late-after-rollout "$LATE" --ready-timeout 600s
"${JOIN_CMD[@]}" 2> "$OUT/late-after-rollout.log" \
|| log "late-after-rollout phase did not run cleanly; see $OUT/late-after-rollout.log"
event late-after-rollout-end