Skip to content

Agent Mesh

node: stop seeking out other members; router: count members, not connections

Antonio Ojea880109ac37f7
diff --git a/cmd/agentmesh-node/main.go b/cmd/agentmesh-node/main.go
index bf685be7..b7ab1c39 100644
--- a/cmd/agentmesh-node/main.go
+++ b/cmd/agentmesh-node/main.go
@@ -458,7 +458,6 @@ func main() {
ControlPlanePubKey: cpPub,
RouterAddrs: rAddrs,
Store: store,
- MeshID: meshFlag,
DiscoveryInterval: discoveryIntervalFlag,
ListenAddrs: listenAddrs,
EnableRelay: enableRelayFlag,
@@ -752,7 +751,6 @@ func main() {
PrivKey: priv,
RouterAddrs: initRouterAddrs,
Store: store,
- MeshID: meshFlag,
DiscoveryInterval: discoveryIntervalFlag,
ListenAddrs: []string{"/ip4/0.0.0.0/udp/0/quic-v1", "/ip4/0.0.0.0/tcp/0"},
EnableRelay: enableRelayFlag,
@@ -865,8 +863,9 @@ func main() {
runCmd.Flags().StringVar(&bindAddrFlag, "bind-addr", "127.0.0.1:8080", "Local TCP address for the HTTP server (MCP and Sidecar API); pass an empty value to serve only on the Unix socket")
runCmd.Flags().StringVar(&socketPathFlag, "socket-path", "", "Unix socket serving the same API, where the socket's owner-only permissions replace the API token (defaults to <data-dir>/"+node.DefaultSocketName+"; pass an empty value to disable)")
runCmd.Flags().StringVar(&metricsAddrFlag, "metrics-addr", "", "Serve Prometheus /metrics, /healthz and /readyz on this address without authentication (e.g. 0.0.0.0:9090), for scrapers and probes that hold no API token; off by default")
- runCmd.Flags().StringVar(&meshFlag, "mesh", node.DefaultMeshName, "Mesh federation name")
- runCmd.Flags().StringVar(&discoveryIntervalFlag, "discovery-interval", node.DefaultDiscoveryInterval, "Polling interval for DHT discovery")
+ runCmd.Flags().StringVar(&meshFlag, "mesh", "public-mesh", "No effect; kept so existing invocations still parse")
+ _ = runCmd.Flags().MarkDeprecated("mesh", "the node no longer announces itself under a mesh name; the flag has no effect")
+ runCmd.Flags().StringVar(&discoveryIntervalFlag, "discovery-interval", node.DefaultDiscoveryInterval, "Delay before the first announcement of this node's services to the DHT, capped at 5s; they are re-announced every 5 minutes after that")
runCmd.Flags().DurationVar(&monitorBootstrapFlag, "monitor-bootstrap", 2*time.Minute, "Initial wait before monitoring router connection")
runCmd.Flags().DurationVar(&monitorCheckIntervalFlag, "monitor-interval", 1*time.Minute, "Interval for checking router connection")
runCmd.Flags().DurationVar(&autoRelayMinIntervalFlag, "autorelay-min-interval", 30*time.Second, "AutoRelay Min Interval")
diff --git a/internal/node/discovery_bootstrap_test.go b/internal/node/discovery_bootstrap_test.go
deleted file mode 100644
index 4ced4e05..00000000
--- a/internal/node/discovery_bootstrap_test.go
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package node
-
-import (
- "context"
- "sync/atomic"
- "testing"
- "time"
-)
-
-// The router lands in the routing table after an asynchronous check that may
-// finish after Start returns; discovery must not advertise into an empty table.
-func TestAwaitRoutingTableReturnsOncePopulated(t *testing.T) {
- var size atomic.Int32
- time.AfterFunc(250*time.Millisecond, func() { size.Store(1) })
-
- start := time.Now()
- awaitRoutingTable(context.Background(), func() int { return int(size.Load()) }, 5*time.Second)
- if waited := time.Since(start); waited < 250*time.Millisecond || waited > 2*time.Second {
- t.Fatalf("returned after %s, want shortly after the table got its peer at 250ms", waited)
- }
-}
-
-func TestAwaitRoutingTableDoesNotWaitWhenPopulated(t *testing.T) {
- start := time.Now()
- awaitRoutingTable(context.Background(), func() int { return 1 }, 5*time.Second)
- if waited := time.Since(start); waited > 50*time.Millisecond {
- t.Fatalf("waited %s on an already populated table", waited)
- }
-}
-
-// With no router ever reachable, discovery must still start: the bound keeps
-// the previous behaviour instead of blocking forever.
-func TestAwaitRoutingTableGivesUpAtBound(t *testing.T) {
- start := time.Now()
- awaitRoutingTable(context.Background(), func() int { return 0 }, 200*time.Millisecond)
- if waited := time.Since(start); waited < 200*time.Millisecond || waited > 2*time.Second {
- t.Fatalf("returned after %s, want about the 200ms bound", waited)
- }
-}
-
-func TestAwaitRoutingTableStopsWithContext(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
- time.AfterFunc(50*time.Millisecond, cancel)
- done := make(chan struct{})
- go func() {
- awaitRoutingTable(ctx, func() int { return 0 }, time.Hour)
- close(done)
- }()
- select {
- case <-done:
- case <-time.After(2 * time.Second):
- t.Fatal("did not return after the context was cancelled")
- }
-}
diff --git a/internal/node/gate_test.go b/internal/node/gate_test.go
index 6d6f26ac..32b5a526 100644
--- a/internal/node/gate_test.go
+++ b/internal/node/gate_test.go
@@ -163,7 +163,6 @@ func startBareNode(t *testing.T, ctx context.Context) (*AgentMeshNode, func()) {
PrivKey: priv,
RouterAddrs: nil,
Store: store,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -358,7 +357,6 @@ func startBareQUICNode(t *testing.T, ctx context.Context) (*AgentMeshNode, func(
PrivKey: priv,
RouterAddrs: nil,
Store: store,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/udp/0/quic-v1"},
EnableRelay: false,
diff --git a/internal/node/node.go b/internal/node/node.go
index cf74175f..e84d2c83 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -52,14 +52,11 @@ import (
records "github.com/libp2p/go-libp2p-kad-dht/records"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/crypto"
- "github.com/libp2p/go-libp2p/core/discovery"
"github.com/libp2p/go-libp2p/core/event"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/core/peerstore"
- "github.com/libp2p/go-libp2p/p2p/discovery/routing"
- "github.com/libp2p/go-libp2p/p2p/discovery/util"
"github.com/libp2p/go-libp2p/p2p/host/autorelay"
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-libp2p/p2p/net/swarm"
@@ -78,10 +75,6 @@ import (
// subsequent discovery or tool calls when dialing unroutable private networks.
const PeerstoreKeyPrivateIPFailed = "private_ip_failed"
-// maxMeshProactiveConnections is the target maximum number of active mesh connections.
-// If active connections count reaches this threshold, the node skips periodic DHT peer discovery.
-const maxMeshProactiveConnections = 30
-
const (
// Cache sizes
RateLimiterSize = 1000
@@ -781,15 +774,6 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
// Listen for Network Evictions/Revocations from the control plane
go n.listenForControlPlaneEvents(ctx)
- interval, err := time.ParseDuration(n.config.DiscoveryInterval)
- if err != nil {
- logger.Warnf("[Discovery] Invalid discovery interval '%s', using default %s: %v", n.config.DiscoveryInterval, DefaultDiscoveryInterval, err)
- interval, _ = time.ParseDuration(DefaultDiscoveryInterval)
- }
-
- // Start DHT Discovery
- go n.startDiscovery(ctx, n.config.MeshID, interval)
-
// Layer 3: Open the Lobby Door (Auth Protocol is bypassed by Layer 4)
n.Host.SetStreamHandler(api.AuthProtocolID, recoverStreamHandler("AuthHandshake", n.HandleAuthHandshake))
n.Host.SetStreamHandler(api.GoAwayProtocolID, recoverStreamHandler("GoAway", func(s network.Stream) { n.HandleGoAway(ctx, s) }))
@@ -1887,94 +1871,6 @@ func (n *AgentMeshNode) validateMeshEvent(_ context.Context, from peer.ID, msg *
return pubsub.ValidationAccept
}
-func (n *AgentMeshNode) startDiscovery(ctx context.Context, meshID string, interval time.Duration) {
- awaitRoutingTable(ctx, n.DHT.RoutingTable().Size, interval)
- routingDiscovery := routing.NewRoutingDiscovery(n.DHT)
- util.Advertise(ctx, routingDiscovery, meshID)
-
- ticker := time.NewTicker(interval)
- defer ticker.Stop()
-
- sem := make(chan struct{}, 8)
-
- for {
- select {
- case <-ctx.Done():
- return
- case <-ticker.C:
- connectedCount := len(n.Host.Network().Peers())
- if connectedCount >= maxMeshProactiveConnections {
- logger.Debugf("[Discovery] Connected peers (%d) >= target (%d). Skipping peer discovery tick.", connectedCount, maxMeshProactiveConnections)
- continue
- }
-
- peers, err := routingDiscovery.FindPeers(ctx, meshID, discovery.Limit(maxMeshProactiveConnections))
- if err != nil {
- logger.Errorf("[Discovery] Failed to find peers: %v", err)
- continue
- }
- for p := range peers {
- // Revoked peers keep advertising the mesh rendezvous, so without
- // this every tick re-dials them and the gater denies it.
- if p.ID == n.Host.ID() || n.peerIsRevoked(p.ID) {
- continue
- }
-
- cond := n.Host.Network().Connectedness(p.ID)
- if cond != network.Connected && cond != network.Limited {
- logger.Debugf("[Discovery] Found peer not connected via DHT: %s (state: %s)", p.ID, cond)
-
- n.Host.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.TempAddrTTL)
- n.preparePeerAddrs(ctx, p.ID)
-
- filteredAddrInfo := n.Host.Peerstore().PeerInfo(p.ID)
- if len(filteredAddrInfo.Addrs) == 0 {
- logger.Debugf("[Discovery] Skipping connection to %s: no valid addresses after filtering", p.ID)
- continue
- }
-
- select {
- case sem <- struct{}{}:
- go func(pi peer.AddrInfo) {
- defer func() { <-sem }()
- dialCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
- defer cancel()
- if err := n.Host.Connect(dialCtx, pi); err != nil {
- logger.Debugf("[Discovery] Failed to connect to %s: %v", pi.ID, err)
- if putErr := n.Host.Peerstore().Put(pi.ID, PeerstoreKeyPrivateIPFailed, true); putErr != nil {
- logger.Errorf("[Discovery] Failed to put peerstore private IP failed key: %v", putErr)
- }
- }
- }(filteredAddrInfo)
- case <-ctx.Done():
- return
- }
- }
- }
- }
- }
-}
-
-// awaitRoutingTable blocks until the DHT routing table has a peer, ctx ends
-// or bound elapses. DHT.Bootstrap only triggers a refresh: the router enters
-// the table after an asynchronous FIND_NODE liveliness check that nothing
-// orders before the auth handshake returns. Advertising into an empty table
-// fails the lookup, and util.Advertise then backs off for two minutes with
-// the node invisible to its peers. Giving up after bound keeps the previous
-// behaviour when no router ever appears.
-func awaitRoutingTable(ctx context.Context, size func() int, bound time.Duration) {
- deadline := time.After(bound)
- for size() == 0 {
- select {
- case <-ctx.Done():
- return
- case <-deadline:
- return
- case <-time.After(100 * time.Millisecond):
- }
- }
-}
-
func (n *AgentMeshNode) getTrustedPublicKeys() []ed25519.PublicKey {
n.keysMu.RLock()
defer n.keysMu.RUnlock()
diff --git a/internal/node/node_test.go b/internal/node/node_test.go
index 3cc081fd..453cacc4 100644
--- a/internal/node/node_test.go
+++ b/internal/node/node_test.go
@@ -32,10 +32,7 @@ import (
dht "github.com/libp2p/go-libp2p-kad-dht"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/host"
- "github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
- "github.com/libp2p/go-libp2p/p2p/discovery/routing"
- "github.com/libp2p/go-libp2p/p2p/discovery/util"
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-msgio"
"github.com/multiformats/go-multiaddr"
@@ -694,7 +691,6 @@ func TestConnectionMonitor_CrashesAfterFailures(t *testing.T) {
PrivKey: priv,
RouterAddrs: nil,
Store: store,
- MeshID: "test",
DiscoveryInterval: "10s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -895,50 +891,3 @@ func TestFindProvidersSkipsRevoked(t *testing.T) {
t.Errorf("revoked peer returned as provider: %v", providers)
}
}
-
-func TestStartDiscoverySkipsRevoked(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- const meshID = "test-mesh"
-
- // Only the bootstrap host runs a server DHT: the others are clients, so
- // nothing dials A and every connection A opens comes from startDiscovery.
- bootstrapHost, _ := newDHTHost(t, dht.ModeServer)
- hostA, dhtA := newDHTHost(t, dht.ModeClient)
- revokedHost, revokedDHT := newDHTHost(t, dht.ModeClient)
- healthyHost, healthyDHT := newDHTHost(t, dht.ModeClient)
-
- for _, h := range []host.Host{hostA, revokedHost, healthyHost} {
- if err := h.Connect(ctx, peer.AddrInfo{ID: bootstrapHost.ID(), Addrs: bootstrapHost.Addrs()}); err != nil {
- t.Fatal(err)
- }
- }
- for _, d := range []*dht.IpfsDHT{dhtA, revokedDHT, healthyDHT} {
- waitForRoutingTable(t, d)
- util.Advertise(ctx, routing.NewRoutingDiscovery(d), meshID)
- }
-
- revokedCache, err := lru.New[string, int64](10)
- if err != nil {
- t.Fatal(err)
- }
- revokedCache.Add(revokedHost.ID().String(), time.Now().UnixMilli())
- node := &AgentMeshNode{Host: hostA, DHT: dhtA, revokedPeers: revokedCache}
-
- go node.startDiscovery(ctx, meshID, 200*time.Millisecond)
-
- // Both advertisers come back in the same FindPeers result, so the healthy
- // one connecting proves the revoked one was offered and skipped.
- deadline := time.Now().Add(20 * time.Second)
- for hostA.Network().Connectedness(healthyHost.ID()) != network.Connected {
- if time.Now().After(deadline) {
- t.Fatalf("healthy peer never connected; discovery did not run")
- }
- time.Sleep(50 * time.Millisecond)
- }
-
- if c := hostA.Network().Connectedness(revokedHost.ID()); c == network.Connected {
- t.Errorf("revoked peer was dialled from discovery (connectedness %s)", c)
- }
-}
diff --git a/internal/node/options.go b/internal/node/options.go
index f86d04aa..1e14327e 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -26,7 +26,6 @@ import (
)
const (
- DefaultMeshName = "public-mesh"
DefaultDiscoveryInterval = "30s"
DefaultConfigFile = "agentmesh-node.yaml"
DefaultRouterConnectTimeout = 5 * time.Second
@@ -68,7 +67,6 @@ type Options struct {
RouterAddrs []multiaddr.Multiaddr
Store *Store
- MeshID string
DiscoveryInterval string
ListenAddrs []string
EnableRelay bool
@@ -177,9 +175,6 @@ type Options struct {
// Default applies default values to Options if they are not specified.
func (o *Options) Default() {
- if o.MeshID == "" {
- o.MeshID = "public-mesh"
- }
if o.DiscoveryInterval == "" {
o.DiscoveryInterval = "30s"
}
diff --git a/internal/node/proxy_test.go b/internal/node/proxy_test.go
index 3b88099a..802f25c2 100644
--- a/internal/node/proxy_test.go
+++ b/internal/node/proxy_test.go
@@ -73,7 +73,6 @@ func TestDatapathIntegration(t *testing.T) {
PrivKey: privA,
RouterAddrs: nil,
Store: storeA,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -100,7 +99,6 @@ func TestDatapathIntegration(t *testing.T) {
PrivKey: privB,
RouterAddrs: nil,
Store: storeB,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -269,7 +267,6 @@ func TestDatapathIntegration_Unauthenticated(t *testing.T) {
PrivKey: privA,
RouterAddrs: nil,
Store: storeA,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -296,7 +293,6 @@ func TestDatapathIntegration_Unauthenticated(t *testing.T) {
PrivKey: privB,
RouterAddrs: nil,
Store: storeB,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -418,7 +414,6 @@ func TestStdioDatapathIntegration(t *testing.T) {
PrivKey: privA,
RouterAddrs: nil,
Store: storeA,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -441,7 +436,6 @@ func TestStdioDatapathIntegration(t *testing.T) {
PrivKey: privB,
RouterAddrs: nil,
Store: storeB,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -589,7 +583,6 @@ func TestDatapathHeadersAndRoutingTable(t *testing.T) {
PrivKey: privA,
RouterAddrs: nil,
Store: storeA,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -616,7 +609,6 @@ func TestDatapathHeadersAndRoutingTable(t *testing.T) {
PrivKey: privB,
RouterAddrs: nil,
Store: storeB,
- MeshID: "test-mesh",
DiscoveryInterval: "1s",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
diff --git a/internal/node/static_service_test.go b/internal/node/static_service_test.go
index 248e8dcf..ba96e5ec 100644
--- a/internal/node/static_service_test.go
+++ b/internal/node/static_service_test.go
@@ -184,7 +184,6 @@ func TestStaticServiceRegistrationRequiresConnection(t *testing.T) {
PrivKey: priv,
RouterAddrs: []multiaddr.Multiaddr{},
Store: store,
- MeshID: "test-mesh",
DiscoveryInterval: "100ms",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
@@ -271,7 +270,6 @@ func TestStaticServiceRegistrationCommandFailure(t *testing.T) {
PrivKey: priv,
RouterAddrs: []multiaddr.Multiaddr{},
Store: store,
- MeshID: "test-mesh",
DiscoveryInterval: "100ms",
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
EnableRelay: false,
diff --git a/internal/router/goaway.go b/internal/router/goaway.go
index 0c71da57..1ba6aff0 100644
--- a/internal/router/goaway.go
+++ b/internal/router/goaway.go
@@ -163,24 +163,24 @@ func (r *Router) inboundConnections() int {
}
// shedIfOverloaded sends members away when the router holds at least its
-// high watermark of inbound connections: as many as take it back to the
-// low watermark, chosen at random so no member is always the one. The
-// connection manager still trims past the high watermark; this runs first
-// and tells the members where they stand, so they do not dial straight
-// back. Returns how many were sent away.
+// high watermark of them: as many as take it back to the low watermark,
+// chosen at random so no member is always the one. Members are counted,
+// not connections: a member attached elsewhere holds a connection here for
+// its DHT queries without being one, and those connections are what the
+// connection manager closes first past the high watermark (see memberTag).
+// Returns how many were sent away.
func (r *Router) shedIfOverloaded(ctx context.Context) int {
- inbound := r.inboundConnections()
- if inbound < r.config.HighWaterMark {
+ members := r.members()
+ if len(members) < r.config.HighWaterMark {
return 0
}
- members := r.members()
- excess := min(inbound-r.config.LowWaterMark, len(members))
+ excess := len(members) - r.config.LowWaterMark
if excess <= 0 {
return 0
}
rand.Shuffle(len(members), func(i, j int) { members[i], members[j] = members[j], members[i] })
shed := members[:excess]
- logger.Warnf("[GoAway] %d inbound connections, high watermark %d: sending %d of %d members away for %s", inbound, r.config.HighWaterMark, len(shed), len(members), DefaultOverloadRetryAfter)
+ logger.Warnf("[GoAway] %d members, high watermark %d: sending %d away for %s", len(members), r.config.HighWaterMark, len(shed), DefaultOverloadRetryAfter)
r.goAwayAll(ctx, shed, &api.RouterGoAway{
Reason: api.RouterGoAway_OVERLOADED,
RetryAfter: durationpb.New(DefaultOverloadRetryAfter),
diff --git a/internal/router/goaway_test.go b/internal/router/goaway_test.go
index 24924c09..cf1bac2c 100644
--- a/internal/router/goaway_test.go
+++ b/internal/router/goaway_test.go
@@ -90,17 +90,18 @@ func newGoAwayRouter(t *testing.T, high int) *Router {
return &Router{Host: h, config: Options{HighWaterMark: high, LowWaterMark: DefaultLowWaterMark(high)}}
}
-// A router at its high watermark sends enough members away to be back at
-// the low one, says why and for how long, and closes their connections
-// after the grace; a peer router is never among them, and a member that
+// A router at its high watermark of members sends enough of them away to be
+// back at the low one, says why and for how long, and closes their
+// connections after the grace; a peer router is never among them, a
+// connection that never authenticated is not counted, and a member that
// does not speak the protocol is closed all the same.
func TestShedIfOverloaded(t *testing.T) {
prev := goAwayGrace
goAwayGrace = 50 * time.Millisecond
t.Cleanup(func() { goAwayGrace = prev })
- // High 4, low 3: at 4 inbound, one member goes.
- r := newGoAwayRouter(t, 4)
+ // High 3, low 2: at 3 members, one goes.
+ r := newGoAwayRouter(t, 3)
peerRouter := newGoAwayListener(t)
connectAuthenticated(t, r, peerRouter.host)
r.peerRouters.Store(peerRouter.host.ID(), true)
@@ -114,12 +115,18 @@ func TestShedIfOverloaded(t *testing.T) {
}
t.Cleanup(func() { _ = mute.Close() })
connectAuthenticated(t, r, mute)
+ // A stranger: connected, never authenticated. Five inbound connections,
+ // three members.
+ stranger := newGoAwayListener(t)
+ if err := stranger.host.Connect(context.Background(), peer.AddrInfo{ID: r.Host.ID(), Addrs: r.Host.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
- if got := r.inboundConnections(); got != 4 {
- t.Fatalf("inbound = %d, want 4", got)
+ if got := r.inboundConnections(); got != 5 {
+ t.Fatalf("inbound = %d, want 5", got)
}
if got := len(r.members()); got != 3 {
- t.Fatalf("members = %d, want 3 (the peer router is not one)", got)
+ t.Fatalf("members = %d, want 3 (the peer router and the stranger are not members)", got)
}
shed := r.shedIfOverloaded(context.Background())
@@ -150,8 +157,11 @@ func TestShedIfOverloaded(t *testing.T) {
if r.Host.Network().Connectedness(peerRouter.host.ID()) != network.Connected || len(peerRouter.received()) != 0 {
t.Fatal("the peer router was sent away")
}
- if r.inboundConnections() != 3 {
- t.Fatalf("inbound after the shed = %d, want 3", r.inboundConnections())
+ if r.Host.Network().Connectedness(stranger.host.ID()) != network.Connected || len(stranger.received()) != 0 {
+ t.Fatal("the stranger was sent away; it was never a member")
+ }
+ if got := len(r.members()); got != 2 {
+ t.Fatalf("members after the shed = %d, want 2", got)
}
if again := r.shedIfOverloaded(context.Background()); again != 0 {
t.Fatalf("below the high watermark the router shed %d more", again)
diff --git a/mobile/agentmesh-node-ffi/ffi/ffi.go b/mobile/agentmesh-node-ffi/ffi/ffi.go
index 4a7cbe34..af04bb5a 100644
--- a/mobile/agentmesh-node-ffi/ffi/ffi.go
+++ b/mobile/agentmesh-node-ffi/ffi/ffi.go
@@ -180,11 +180,6 @@ func StartNode(configJSON string) error {
discoveryInterval = node.DefaultDiscoveryInterval
}
- meshID := config.MeshID
- if meshID == "" {
- meshID = node.DefaultMeshName
- }
-
// Create and initialize the node
var services []api.ServiceConfig
for _, svc := range config.Services {
@@ -213,7 +208,6 @@ func StartNode(configJSON string) error {
ControlPlanePubKey: controlPlanePubKey,
RouterAddrs: routerAddrs,
Store: store,
- MeshID: meshID,
DiscoveryInterval: discoveryInterval,
ListenAddrs: listenAddrs,
EnableRelay: config.EnableRelay,
diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.md
index 620d841a..82f580f7 100644
--- a/site/content/docs/guides/scalability.md
+++ b/site/content/docs/guides/scalability.md
@@ -17,11 +17,14 @@ stayed through a rollout of every router and both control plane replicas.
A member is one `agentmesh-node` process. It holds a session with two
routers (`--routers`), chosen from those the control plane lists, and a
relay reservation on each. Through those sessions it keeps a GossipSub
-subscription for control plane events and a DHT client. It opens short
-connections to other routers when a DHT query or a call takes it there,
-and to the router of a peer it calls, where it runs the credential
-handshake before asking for the circuit. At rest it uses about 55 MiB of
-memory, 300 goroutines and 20 file descriptors, and almost no CPU.
+subscription for control plane events. Its DHT client also keeps a
+connection to the other routers, which carries no session: a router
+counts members, not connections, against its watermarks, and those are the
+connections it closes first when it is full. A member connects to another
+member only to call it, through that member's router, where it runs the
+credential handshake before asking for the circuit. At rest it uses about
+55 MiB of memory, 300 goroutines and 20 file descriptors, and almost no
+CPU.
Joining is the expensive moment. The member enrolls at the control plane
(one signed request, one database write, one credential minted), reads the
@@ -41,7 +44,7 @@ them.
| Bound | Where | Default | At the bound |
|---|---|---|---|
| Routers per member | `agentmesh-node --routers` | 2 | A member holds this many routers; one is enough to be on the mesh, the second covers a router restart. |
-| Connections per router | `agentmesh-router --high-watermark`, `--low-watermark` | 4000, 15% below (3400) | At the high mark the router sends members away, at random, down to the low mark; they attach to another router and stay off this one for five minutes. A mesh of N routers holds about N × 4000 / 2 members. |
+| Connections per router | `agentmesh-router --high-watermark`, `--low-watermark` | 4000, 15% below (3400) | At the high mark of members the router sends members away, at random, down to the low mark; they attach to another router and stay off this one for five minutes. Above the high mark of connections, the connection manager closes those that carry no session first. A mesh of N routers holds about N × 4000 / 2 members. |
| Connections per source address | `agentmesh-router --conns-per-source-ip` | a quarter of the high mark (1000) | The router refuses the next connection from that address. Members behind one NAT or one cluster's egress share an address. |
| Relay reservations per router | `agentmesh-router --relay-max-reservations` | the high mark (4000) | A member that cannot reserve is not reachable through that router. |
| Relayed connections per peer | `agentmesh-router --relay-max-circuits` | 1024 | A service called through the relay holds one circuit per caller; the next caller is refused. |
diff --git a/site/content/docs/reference/agentmesh-node.md b/site/content/docs/reference/agentmesh-node.md
index 83a03c22..8d8a1c22 100644
--- a/site/content/docs/reference/agentmesh-node.md
+++ b/site/content/docs/reference/agentmesh-node.md
@@ -114,8 +114,7 @@ explains how to enroll.
| `--router-selector` | none | Attach only to routers whose credential attests this `key=value` label, repeatable, every pair required: `--router-selector region=eu`. The labels are the ones the control plane lists on `/info`, attested at the router's enrollment. A selector no router attests leaves the node retrying as if no router answered, with a warning naming the selector and the labels it saw. |
| `--router-prefer` | none | Attach to routers attesting this `key=value` label before others, repeatable; a router attesting more of the pairs comes earlier, and among equals the one with the most room. `--router-prefer zone=eu-a` keeps a node on its zone's routers while they have capacity and lets it fall to the others. |
| `--enable-relay` | `false` | Relay traffic for other peers. |
-| `--mesh` | `public-mesh` | Mesh federation name. Must match the routers' name. |
-| `--discovery-interval` | `30s` | DHT discovery polling interval. |
+| `--discovery-interval` | `30s` | Delay before the first announcement of this node's services to the DHT, capped at 5 s; services are re-announced every 5 minutes after that. |
| `--dht-lookup-limit` | `20` | Providers to query per DHT lookup. |
| `--discovery-concurrency` | `10` | Concurrent catalog fetches during discovery. |
| `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. |
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index 67621ff5..64e2caa4 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -42,7 +42,7 @@ replica count.
| `--lease-renew-interval` | `300s` | How often the lease is renewed. Must be well below the control plane's `--lease-duration`. |
| `--allow-loopback` | `false` | Announce and accept loopback and link-local addresses. For a router and nodes on one host. |
| `--conns-per-source-ip` | a quarter of `--high-watermark` (`1000`) | Inbound connections accepted per source address. One address can hold at most this share of the router's connection budget, so filling a router takes at least four addresses. Members behind a NAT or a cluster's SNAT share one address and each holds one connection per router, two while enrolling. |
-| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | Connection limits. Every member holds one connection, so the high mark is how many members the router holds. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The connection manager closes connections above the high mark as well, oldest first among those that never authenticated (a control plane publisher, a member attached elsewhere running a DHT query, a stranger), then members; connections to the other routers are never closed. |
+| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | How many members the router holds: peers that passed the handshake, other routers aside. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The same numbers bound connections: above the high mark the connection manager closes connections, oldest first among those that carry no session (a member attached elsewhere whose DHT client is connected here, a control plane publisher, a stranger), then members; connections to the other routers are never closed. |
| `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. |
| `--dht-max-record-age` | library default | DHT value record lifetime. |
| `--relay-limit-duration`, `--relay-limit-data` | `1h`, `0` | Caps on each relayed connection: lifetime, and bytes per direction (`512MiB`, `1GB`). The relay cuts the connection when either is reached. `0` means no limit. |
@@ -64,14 +64,14 @@ replica count.
4. Runs the mutual credential handshake on every inbound connection and
refuses peers whose credential does not verify and peers on the ban list.
5. Refreshes its own credential before it expires, like a node.
-6. Every 5 seconds, compares its inbound connections with
- `--high-watermark`. At the mark it picks members at random, as many as
- take it back to `--low-watermark`, and sends each a go-away on
- `/mesh/goaway/1.0.0` with the reason `OVERLOADED` and a retry time of
- 5 minutes. A member that takes the message attaches to another router
- at once and stays off this one for that time; one that does not is
- closed after a 5 second grace, like any other. Connections to the other
- routers are never shed: the DHT runs over them.
+6. Every 5 seconds, compares the members it holds, the peers that passed
+ the handshake, with `--high-watermark`. At the mark it picks members at
+ random, as many as take it back to `--low-watermark`, and sends each a
+ go-away on `/mesh/goaway/1.0.0` with the reason `OVERLOADED` and a
+ retry time of 5 minutes. A member that takes the message attaches to
+ another router at once and stays off this one for that time; one that
+ does not is closed after a 5 second grace, like any other. Connections
+ to the other routers are never shed: the DHT runs over them.
7. On `SIGTERM` or `SIGINT`, sends a last lease with `--shutdown-lease-ttl`,
sends every member a go-away with the reason `DRAINING` and that ttl as
the retry time, and after the same grace closes its connections. The
diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.go
index d489435b..4b5baabf 100644
--- a/tests/integration/router_attachment_test.go
+++ b/tests/integration/router_attachment_test.go
@@ -38,9 +38,33 @@ import (
// routerFixture is one agentmesh-router the test started, with the key it
// was given so its peer ID is known before it runs.
type routerFixture struct {
- cmd *exec.Cmd
- peerID peer.ID
- port int
+ cmd *exec.Cmd
+ peerID peer.ID
+ port int
+ metricsAddr string
+}
+
+// gauge reads one metric from the router's /metrics, by its full name with
+// labels, and fails when it is not there.
+func (r *routerFixture) gauge(t *testing.T, name string) float64 {
+ t.Helper()
+ resp, err := http.Get("http://" + r.metricsAddr + "/metrics")
+ if err != nil {
+ t.Fatalf("scrape %s: %v", r.metricsAddr, err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ body, _ := io.ReadAll(resp.Body)
+ for _, line := range strings.Split(string(body), "\n") {
+ if strings.HasPrefix(line, name+" ") {
+ var v float64
+ if _, err := fmt.Sscanf(strings.TrimPrefix(line, name+" "), "%g", &v); err != nil {
+ t.Fatalf("parse %q: %v", line, err)
+ }
+ return v
+ }
+ }
+ t.Fatalf("metric %s not served by %s", name, r.metricsAddr)
+ return 0
}
// startLabelledRouter runs a router that declares the given labels and
@@ -64,6 +88,7 @@ func startLabelledRouter(t *testing.T, routerBin, tmpDir, name string, cpPort in
t.Fatal(err)
}
port := getFreePort(t)
+ metricsAddr := fmt.Sprintf("127.0.0.1:%d", getFreePort(t))
args := []string{
"--control-plane", fmt.Sprintf("http://127.0.0.1:%d", cpPort),
"--listen", fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", port),
@@ -71,6 +96,7 @@ func startLabelledRouter(t *testing.T, routerBin, tmpDir, name string, cpPort in
"--allow-loopback",
"--oidc-token", routerJWT,
"--lease-renew-interval", "250ms",
+ "--metrics-addr", metricsAddr,
}
for _, l := range labels {
args = append(args, "--label", l)
@@ -94,7 +120,7 @@ func startLabelledRouter(t *testing.T, routerBin, tmpDir, name string, cpPort in
t.Logf("--- %s ---\n%s", name, out)
}
})
- return &routerFixture{cmd: cmd, peerID: peerID, port: port}
+ return &routerFixture{cmd: cmd, peerID: peerID, port: port, metricsAddr: metricsAddr}
}
// routersConnectedTo lists the routers whose last lease reported a
@@ -113,18 +139,28 @@ func routersConnectedTo(t *testing.T, cpPort int, peerID peer.ID) []string {
return out
}
-// attachedRouters reads the routers the node holds as relays from its own
-// debug endpoint.
-func attachedRouters(t *testing.T, n *backgroundNode) []string {
+// meshInfo reads the node's own view: the routers it holds and every peer
+// it has a connection with.
+func meshInfo(t *testing.T, n *backgroundNode) (attached, connected []string) {
t.Helper()
var info struct {
AttachedRouters []string `json:"attached_routers"`
+ ConnectedPeers []string `json:"connected_peers"`
}
if err := json.Unmarshal([]byte(debugGetWithToken(t, n.apiAddr, n.token, "/debug/mesh-info")), &info); err != nil {
t.Fatalf("decode mesh-info: %v", err)
}
slices.Sort(info.AttachedRouters)
- return info.AttachedRouters
+ slices.Sort(info.ConnectedPeers)
+ return info.AttachedRouters, info.ConnectedPeers
+}
+
+// attachedRouters reads the routers the node holds as relays from its own
+// debug endpoint.
+func attachedRouters(t *testing.T, n *backgroundNode) []string {
+ t.Helper()
+ attached, _ := meshInfo(t, n)
+ return attached
}
// debugGetWithToken is debugGet for a node whose API token is not the
@@ -444,3 +480,73 @@ func mustPeerID(t *testing.T, s string) peer.ID {
}
return id
}
+
+// TestIdleNodesHoldOnlyTheirRouters pins what a member costs the mesh when
+// it is not calling anyone: a session with the routers it chose and nothing
+// else. Three routers, three nodes each pinned to its own router, no
+// service and no call; after the discovery interval has ticked many times,
+// no node is connected to another node, and each router has authenticated
+// its own node and the two other routers, nobody else. A node that sought
+// out other members on its own would be connected to them, through their
+// routers, and would have authenticated at every router on the way, which
+// is the load of a member on every router rather than on two. Connections
+// a node's DHT client holds to the other routers are allowed: they carry
+// no session and are the first a router closes when it is full.
+func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {
+ m := startAttachmentMesh(t)
+ sites := []string{"a", "b", "c"}
+ routers := map[string]*routerFixture{}
+ for _, s := range sites {
+ routers[s] = m.startRouter(t, "router-"+s, "site="+s)
+ }
+ // Federation on the first pass: every router finds the ones before it.
+ waitForEqual(t, "routers connected to router c", func() []string { return routersConnectedTo(t, m.cpPort, routers["c"].peerID) },
+ peerIDStrings(routers["a"].peerID, routers["b"].peerID), 10*time.Second)
+
+ nodes := map[string]*backgroundNode{}
+ for _, s := range sites {
+ nodes[s] = launchNode(t, m.nodeBin, m.nodeEnv("node-"+s), filepath.Join(m.tmpDir, "node-"+s), append(m.nodeArgs(),
+ "--api-token-path", tokenPath(t, "token-"+s),
+ "--allow-loopback",
+ "--router-selector", "site="+s,
+ "--monitor-bootstrap", "500ms",
+ "--monitor-interval", "500ms",
+ )...)
+ }
+ for _, s := range sites {
+ nodes[s].waitForAPI(t)
+ waitForEqual(t, "routers attached by node "+s, func() []string { return attachedRouters(t, nodes[s]) }, peerIDStrings(routers[s].peerID), 10*time.Second)
+ }
+
+ // Twenty discovery intervals (200ms) with nothing to do.
+ time.Sleep(4 * time.Second)
+
+ for _, s := range sites {
+ attached, connected := meshInfo(t, nodes[s])
+ if !slices.Equal(attached, peerIDStrings(routers[s].peerID)) {
+ t.Errorf("node %s holds %v, want its router %s", s, attached, routers[s].peerID)
+ }
+ for _, p := range connected {
+ for _, other := range sites {
+ if other != s && p == nodes[other].peerID.String() {
+ t.Errorf("idle node %s is connected to node %s", s, other)
+ }
+ }
+ }
+ // Its own node and the two peer routers.
+ if got := routers[s].gauge(t, "agentmesh_router_authenticated_peers"); got != 3 {
+ t.Errorf("router %s authenticated %v peers, want 3 (its node and two routers)", s, got)
+ }
+ if t.Failed() {
+ t.Logf("node %s connected peers: %v", s, connected)
+ }
+ }
+ if t.Failed() {
+ for _, s := range sites {
+ out := nodes[s].log()
+ if i := strings.LastIndex(out, "[AuthN] Successfully authenticated"); i > 0 {
+ t.Logf("--- node %s, last handshake line ---\n%s", s, out[i:min(len(out), i+300)])
+ }
+ }
+ }
+}