Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node, bench: /readyz tells the truth on every listener; a member binds no loopback port
Antonio Ojea6e308ff3c91d
diff --git a/cmd/agentmesh-bench/main.go b/cmd/agentmesh-bench/main.goindex 3968e77b..01b69d1a 100644--- a/cmd/agentmesh-bench/main.go+++ b/cmd/agentmesh-bench/main.go@@ -282,7 +282,6 @@ func newJoinCmd() *cobra.Command {flags.DurationVar(&opts.CallTimeout, "call-timeout", 120*time.Second, "Bound on one member's discovery and first call, from readiness")flags.DurationVar(&opts.Hold, "hold", 0, "Keep the fleet resident this long after the journeys end, sampling readiness; 0 tears it down at once")flags.DurationVar(&opts.SampleInterval, "sample-interval", 10*time.Second, "How often each resident member's readiness is read during --hold")- flags.IntVar(&opts.MetricsBasePort, "metrics-base-port", 20000, "Member i serves /readyz on 127.0.0.1:(port+i)")flags.StringArrayVar(&opts.NodeArgs, "node-arg", nil, "Extra argument for every agentmesh-node, repeatable (e.g. --node-arg=--insecure-control-plane)")flags.StringVar(&marker, "resident-marker", "", "File to create once the fleet is resident and the hold has begun")flags.StringArrayVar(&scrape, "scrape", nil, "Metrics endpoint to record before and after the run, repeatable")diff --git a/internal/bench/join.go b/internal/bench/join.goindex 91eceb64..ac7904eb 100644--- a/internal/bench/join.go+++ b/internal/bench/join.go@@ -93,10 +93,6 @@ type JoinOptions struct {// SampleInterval is how often each resident member is read during Hold.SampleInterval time.Duration- // MetricsBasePort is the first of Count consecutive loopback ports;- // member i serves /readyz on MetricsBasePort+i.- MetricsBasePort int-// NodeArgs are appended to every member's command line as given.NodeArgs []string@@ -208,6 +204,7 @@ const (// and one that takes a minute is not asked a thousand times.readyPollMin = 50 * time.MillisecondreadyPollMax = 500 * time.Millisecond+ readyTimeout = 2 * time.SecondcallRetryInterval = 5 * time.SecondrequestTimeout = 30 * time.SecondteardownGrace = 10 * time.Second@@ -222,17 +219,19 @@ const (const mcpInitialize = `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"agentmesh-bench","version":"0"}}}`type member struct {- index int- dir string- socket string- metricsAddr string- readyz string- logPath string- cmd *exec.Cmd- logFile *os.File- started time.Time- exited chan struct{}- exitErr error+ index int+ dir string+ socket string+ // client speaks to the member over its Unix socket: readiness, discovery+ // and the first call. A member binds no loopback port, so a fleet of any+ // size cannot collide with the kernel's ephemeral range.+ client *http.Client+ logPath string+ cmd *exec.Cmd+ logFile *os.File+ started time.Time+ exited chan struct{}+ exitErr errorreadyAt time.TimecalledAt time.Time@@ -364,9 +363,6 @@ func (o *JoinOptions) setDefaults() {if o.SampleInterval <= 0 {o.SampleInterval = 10 * time.Second}- if o.MetricsBasePort <= 0 {- o.MetricsBasePort = 20000- }}func (o *JoinOptions) logf(format string, args ...any) {@@ -378,14 +374,13 @@ func (o *JoinOptions) logf(format string, args ...any) {func newMember(opts JoinOptions, i int) *member {dir := filepath.Join(opts.Dir, fmt.Sprintf("member-%04d", i))- metricsAddr := fmt.Sprintf("127.0.0.1:%d", opts.MetricsBasePort+i)+ socket := filepath.Join(dir, "node.sock")return &member{- index: i,- dir: dir,- socket: filepath.Join(dir, "node.sock"),- metricsAddr: metricsAddr,- readyz: "http://" + metricsAddr + "/readyz",- logPath: filepath.Join(dir, "node.log"),+ index: i,+ dir: dir,+ socket: socket,+ client: unixClient(socket),+ logPath: filepath.Join(dir, "node.log"),}}@@ -407,7 +402,6 @@ func (m *member) start(opts JoinOptions) error {"--data-dir", m.dir,"--socket-path", m.socket,"--bind-addr=",- "--metrics-addr", m.metricsAddr,"--listen", "/ip4/0.0.0.0/tcp/0","--listen", "/ip4/0.0.0.0/udp/0/quic-v1",}@@ -456,11 +450,10 @@ func journey(ctx context.Context, opts JoinOptions, m *member) {// waitReady polls the member's /readyz, which answers 200 only once the node// holds an authenticated connection to a router.func (m *member) waitReady(ctx context.Context, timeout time.Duration) error {- client := &http.Client{Timeout: 2 * time.Second}deadline := time.Now().Add(timeout)interval := readyPollMinfor {- if isReady(ctx, client, m.readyz) {+ if m.isReady(ctx) {m.readyAt = time.Now()return nil}@@ -481,12 +474,14 @@ func (m *member) waitReady(ctx context.Context, timeout time.Duration) error {}}-func isReady(ctx context.Context, client *http.Client, url string) bool {- req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)+func (m *member) isReady(ctx context.Context) bool {+ ctx, cancel := context.WithTimeout(ctx, readyTimeout)+ defer cancel()+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost/readyz", nil)if err != nil {return false}- resp, err := client.Do(req)+ resp, err := m.client.Do(req)if err != nil {return false}@@ -500,7 +495,7 @@ func isReady(ctx context.Context, client *http.Client, url string) bool {// deadline: a provider record can name a peer a rollout just replaced, and a// member that joined seconds ago has a thin routing table.func (m *member) firstCall(ctx context.Context, opts JoinOptions) error {- client := unixClient(m.socket)+ client := m.clientdefer client.CloseIdleConnections()deadline := m.readyAt.Add(opts.CallTimeout)lastErr := "none discovered"@@ -691,12 +686,11 @@ func hold(ctx context.Context, opts JoinOptions, members []*member) *HoldReportreport := &HoldReport{}var outages []time.Duration- client := &http.Client{Timeout: 2 * time.Second}start := time.Now()end := start.Add(opts.Hold)for {now := time.Now()- ready := sampleReady(ctx, client, members)+ ready := sampleReady(ctx, members)count := 0for i, m := range members {if ready[i] {@@ -759,7 +753,7 @@ func hold(ctx context.Context, opts JoinOptions, members []*member) *HoldReport}// sampleReady reads every member's /readyz, a bounded number at a time.-func sampleReady(ctx context.Context, client *http.Client, members []*member) []bool {+func sampleReady(ctx context.Context, members []*member) []bool {ready := make([]bool, len(members))sem := make(chan struct{}, sampleConcurrency)var wg sync.WaitGroup@@ -772,7 +766,7 @@ func sampleReady(ctx context.Context, client *http.Client, members []*member) []go func() {defer wg.Done()defer func() { <-sem }()- ready[i] = isReady(ctx, client, m.readyz)+ ready[i] = m.isReady(ctx)}()}wg.Wait()diff --git a/internal/bench/join_test.go b/internal/bench/join_test.goindex 1ee3306a..1191efb1 100644--- a/internal/bench/join_test.go+++ b/internal/bench/join_test.go@@ -31,8 +31,11 @@ import (// The members RunJoin starts are whatever binary it is pointed at. Here that// is this test binary again, re-entered through TestMain as a node that-// answers /readyz and the two sidecar routes the journey uses, with its-// behaviour set through the environment the children inherit.+// answers /readyz and the two sidecar routes the journey uses on its Unix+// socket, with its behaviour set through the environment the children+// inherit. Like the real node it binds no loopback port unless told to, and+// it refuses to be told: a fleet of a thousand on one host collided with the+// kernel's ephemeral range when every member had one.func TestMain(m *testing.M) {if os.Getenv("AGENTMESH_BENCH_FAKE_NODE") == "1" {@@ -42,11 +45,14 @@ func TestMain(m *testing.M) {}func fakeNode(args []string) int {- var metricsAddr, socket, tokenPath string+ var socket, tokenPath stringfor i := 0; i < len(args)-1; i++ {switch args[i] {- case "--metrics-addr":- metricsAddr = args[i+1]+ case "--metrics-addr", "--bind-addr":+ if args[i+1] != "" {+ fmt.Fprintln(os.Stderr, "fake node: a member must not bind a loopback port:", args[i], args[i+1])+ return 4+ }case "--socket-path":socket = args[i+1]case "--bootstrap-token-path":@@ -78,15 +84,14 @@ func fakeNode(args []string) int {return true}- metrics := http.NewServeMux()- metrics.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {+ sidecar := http.NewServeMux()+ sidecar.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {if !ready() {http.Error(w, "not ready", http.StatusServiceUnavailable)return}w.WriteHeader(http.StatusOK)})- sidecar := http.NewServeMux()sidecar.HandleFunc("/mesh/service/discover", func(w http.ResponseWriter, _ *http.Request) {var out []map[string]stringfor _, p := range providers {@@ -104,17 +109,11 @@ func fakeNode(args []string) int {_, _ = fmt.Fprint(w, `{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-06-18","serverInfo":{"name":"fake","version":"0"}}}`)})- tcp, err := net.Listen("tcp", metricsAddr)- if err != nil {- fmt.Fprintln(os.Stderr, "fake node:", err)- return 1- }unix, err := net.Listen("unix", socket)if err != nil {fmt.Fprintln(os.Stderr, "fake node:", err)return 1}- go func() { _ = http.Serve(tcp, metrics) }() //nolint:gosec // a test doublego func() { _ = http.Serve(unix, sidecar) }() //nolint:gosec // a test doublestop := make(chan os.Signal, 1)signal.Notify(stop, syscall.SIGTERM, os.Interrupt)@@ -152,37 +151,9 @@ func fakeFleet(t *testing.T, count int, env map[string]string) JoinOptions {ControlPlane: "http://127.0.0.1:1",BootstrapTokenPath: token,Service: "everything",- MetricsBasePort: freePortBase(t, count),}}-// freePortBase finds count consecutive free loopback ports.-func freePortBase(t *testing.T, count int) int {- t.Helper()- for attempt := 0; attempt < 20; attempt++ {- l, err := net.Listen("tcp", "127.0.0.1:0")- if err != nil {- t.Fatal(err)- }- base := l.Addr().(*net.TCPAddr).Port- _ = l.Close()- free := true- for i := 1; i < count; i++ {- probe, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i))- if err != nil {- free = false- break- }- _ = probe.Close()- }- if free {- return base- }- }- t.Fatal("no run of free ports")- return 0-}-func TestRunJoinTimesEachStageAndRetriesStaleProviders(t *testing.T) {// A provider record that names a peer which does not answer is the// normal state of a mesh just after a rollout. The journey has to getdiff --git a/internal/node/sidecar.go b/internal/node/sidecar.goindex 79f3fd7b..00ba4b40 100644--- a/internal/node/sidecar.go+++ b/internal/node/sidecar.go@@ -47,7 +47,7 @@ func StartSidecarServer(node *AgentMeshNode, addr, socketPath, token, certFile,// Public endpointsmux.HandleFunc("/healthz", handleHealthz)- mux.HandleFunc("/readyz", handleReadyz)+ mux.HandleFunc("/readyz", handleReadyzFor(node))mux.HandleFunc("/.well-known/oauth-protected-resource", func(w http.ResponseWriter, r *http.Request) {handleOAuthProtectedResource(node, w, r)})@@ -315,7 +315,7 @@ func StartUnauthSidecarServer(controlPlaneURL, addr, socketPath, certFile, keyFi// Public endpointsmux.HandleFunc("/healthz", handleHealthz)- mux.HandleFunc("/readyz", handleReadyz)+ mux.HandleFunc("/readyz", handleReadyzFor(nil))// Mount Unauthenticated MCP handlermcpHandler := NewUnauthenticatedMCPHandler(controlPlaneURL)@@ -383,10 +383,20 @@ func handleHealthz(w http.ResponseWriter, r *http.Request) {}}-func handleReadyz(w http.ResponseWriter, r *http.Request) {- w.WriteHeader(http.StatusOK)- if _, err := w.Write([]byte("OK")); err != nil {- logger.Errorf("Failed to write response: %v", err)+// handleReadyzFor answers 200 only once the node holds an authenticated+// connection to a router, the same answer the metrics listener gives, so a+// probe or a harness reading either listener learns the same thing. A node+// without an identity yet (nil) is not ready.+func handleReadyzFor(node *AgentMeshNode) http.HandlerFunc {+ return func(w http.ResponseWriter, _ *http.Request) {+ if node == nil || node.debugReady() != nil || !node.IsConnected() {+ http.Error(w, "not connected to the mesh", http.StatusServiceUnavailable)+ return+ }+ w.WriteHeader(http.StatusOK)+ if _, err := w.Write([]byte("OK")); err != nil {+ logger.Errorf("Failed to write response: %v", err)+ }}}diff --git a/internal/node/sidecar_auth_test.go b/internal/node/sidecar_auth_test.goindex b3346524..34405b53 100644--- a/internal/node/sidecar_auth_test.go+++ b/internal/node/sidecar_auth_test.go@@ -19,6 +19,9 @@ import ("path/filepath""testing""time"++ "github.com/libp2p/go-libp2p"+ "github.com/libp2p/go-libp2p/core/peer")// TestConstantTimeEqual is a correctness check, not a timing one: a timing@@ -57,6 +60,80 @@ func TestConstantTimeEqual(t *testing.T) {}}+// TestSidecarReadyzFollowsTheMeshConnection pins that the sidecar's /readyz+// says what the metrics listener's does: 503 until the node holds an+// authenticated, connected router, then 200, on the socket and on TCP. A+// harness that reads readiness over the socket, as agentmesh-bench does so+// that a member binds no loopback port, learns the same thing a kubelet+// probing the metrics port would.+func TestSidecarReadyzFollowsTheMeshConnection(t *testing.T) {+ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = store.Close() }()+ host, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = host.Close() }()+ node := &AgentMeshNode{+ Host: host,+ Store: store,+ BiscuitTimeout: 500 * time.Millisecond,+ services: NewServiceRegistry(&fakeDHT{}, 0),+ }+ node.Directory = &routerDirectory{host: host, routers: node.attachedRouters, addrs: host.Addrs}+ socketPath := filepath.Join(t.TempDir(), "agentmesh.sock")+ srv, err := StartSidecarServer(node, "127.0.0.1:0", socketPath, "test-token", "", "", "")+ if err != nil {+ t.Fatalf("failed to start sidecar server: %v", err)+ }+ defer func() { _ = srv.Close() }()+ socket := waitForSocket(t, socketPath)+ tcp := &http.Client{Timeout: 2 * time.Second}++ readyz := func(t *testing.T, client *http.Client, url string) int {+ t.Helper()+ resp, err := client.Get(url)+ if err != nil {+ t.Fatalf("GET %s: %v", url, err)+ }+ defer func() { _ = resp.Body.Close() }()+ return resp.StatusCode+ }+ for name, url := range map[string]string{"socket": "http://localhost/readyz", "tcp": "http://" + node.BoundHTTPAddr + "/readyz"} {+ client := socket+ if name == "tcp" {+ client = tcp+ }+ if code := readyz(t, client, url); code != http.StatusServiceUnavailable {+ t.Errorf("%s /readyz = %d with no authenticated router, want 503", name, code)+ }+ }++ routerHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = routerHost.Close() }()+ if err := host.Connect(t.Context(), peer.AddrInfo{ID: routerHost.ID(), Addrs: routerHost.Addrs()}); err != nil {+ t.Fatal(err)+ }+ node.mu.Lock()+ node.authenticatedRouters = map[peer.ID]bool{routerHost.ID(): true}+ node.mu.Unlock()+ for name, url := range map[string]string{"socket": "http://localhost/readyz", "tcp": "http://" + node.BoundHTTPAddr + "/readyz"} {+ client := socket+ if name == "tcp" {+ client = tcp+ }+ if code := readyz(t, client, url); code != http.StatusOK {+ t.Errorf("%s /readyz = %d once a router is authenticated and connected, want 200", name, code)+ }+ }+}+// TestMetricsGatedOnTCPButNotOnTheSocket pins where the metrics endpoint sits in// the sidecar's trust model. Its labels carry peer IDs and per-peer request// counts, and the TCP listener is reachable by any local process, so it isdiff --git a/internal/node/sidecar_test.go b/internal/node/sidecar_test.goindex 17b55e8a..0f25ea78 100644--- a/internal/node/sidecar_test.go+++ b/internal/node/sidecar_test.go@@ -399,7 +399,8 @@ func TestSidecarServerAuthEnforcement(t *testing.T) {needsToken bool}{{"healthz is public", "GET", "/healthz", http.StatusOK, false},- {"readyz is public", "GET", "/readyz", http.StatusOK, false},+ // Public, and honest: this node holds no router, so it is not ready.+ {"readyz is public", "GET", "/readyz", http.StatusServiceUnavailable, false},// Liveness says nothing; metrics name peers and count their traffic.{"metrics is protected", "GET", "/metrics", http.StatusUnauthorized, false},// Services are declared in configuration only: the former runtimediff --git a/tests/integration/member_burst_test.go b/tests/integration/member_burst_test.goindex e5940169..d1a590bb 100644--- a/tests/integration/member_burst_test.go+++ b/tests/integration/member_burst_test.go@@ -16,7 +16,6 @@ package integration_testimport ("context"- "fmt""net""net/http/httptest""os"@@ -92,7 +91,6 @@ func TestMemberBurstJoinsConcurrently(t *testing.T) {Service: "everything",ReadyTimeout: 30 * time.Second,CallTimeout: 30 * time.Second,- MetricsBasePort: reservePortBlock(t, members),NodeArgs: []string{"--allow-loopback"},Log: testLogWriter{t},})@@ -134,29 +132,6 @@ func TestMemberBurstJoinsConcurrently(t *testing.T) {members, report.ReadyMs.P50, report.ReadyMs.P95, report.JourneyMs.P50, report.JourneyMs.P95, report.JourneyMs.Max, report.CallAttempts)}-// reservePortBlock finds n consecutive free loopback ports in the reserved-// range and returns the first.-func reservePortBlock(t *testing.T, n int) int {- t.Helper()- for range 50 {- base := getFreePort(t)- free := true- for i := 1; i < n; i++ {- l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", base+i))- if err != nil {- free = false- break- }- _ = l.Close()- }- if free {- return base- }- }- t.Fatalf("no run of %d free ports", n)- return 0-}-// logTail returns the last part of a log file, for a failure message.func logTail(path string) string {data, err := os.ReadFile(path) // #nosec G304 -- a log the test wrotediff --git a/tests/scale/member-journey.sh b/tests/scale/member-journey.shindex 2c67d100..ee38e309 100755--- a/tests/scale/member-journey.sh+++ b/tests/scale/member-journey.sh@@ -259,16 +259,15 @@ summarize() {# a backgrounded function is a subshell, and a signal to it would not reach# the run inside.JOIN_CMD=()-join_cmd() { # phase count base-port extra...- local phase="$1" count="$2" base="$3" a- shift 3+join_cmd() { # phase count extra...+ local phase="$1" count="$2" a+ shift 2JOIN_CMD=("$AGENTMESH_BENCH" join--node-bin "$AGENTMESH_NODE"--control-plane "$CONTROL_PLANE"--bootstrap-token-path "$BOOTSTRAP_TOKEN_PATH"--service "$SERVICE"--count "$count"- --metrics-base-port "$base"--dir "$OUT/$phase"--label "phase=$phase" --label "count=$count" --label "burst=$COUNT" --label "host=$(hostname)"--out "$OUT/$phase.json")@@ -288,7 +287,7 @@ rm -f "$OUT/resident"snapshot beforeevent burst-startlog "burst: ${COUNT} members at ${RAMP}/s, then holding ${HOLD}"-join_cmd burst "$COUNT" 20000 --ramp "$RAMP" --hold "$HOLD" --resident-marker "$OUT/resident"+join_cmd burst "$COUNT" --ramp "$RAMP" --hold "$HOLD" --resident-marker "$OUT/resident""${JOIN_CMD[@]}" 2> "$OUT/burst.log" &JOIN_PID=$!while [[ ! -f "$OUT/resident" ]]; do@@ -309,7 +308,7 @@ PYif (( LATE > 0 )); thenevent late-startlog "late: ${LATE} members join the populated mesh"- join_cmd late "$LATE" 30000+ join_cmd late "$LATE""${JOIN_CMD[@]}" 2> "$OUT/late.log" || log "late phase did not run cleanly; see $OUT/late.log"event late-endfi@@ -326,7 +325,7 @@ if (( ROLLOUT )); then# timer; a long bound here measures that wait instead of hiding it.event late-after-rollout-startlog "late: ${LATE} members join behind the rollout"- join_cmd late-after-rollout "$LATE" 31000 --ready-timeout 600s+ join_cmd late-after-rollout "$LATE" --ready-timeout 600s"${JOIN_CMD[@]}" 2> "$OUT/late-after-rollout.log" \|| log "late-after-rollout phase did not run cleanly; see $OUT/late-after-rollout.log"event late-after-rollout-end