Skip to content

Agent Mesh

node: attach to two routers, chosen by label and load

Antonio Ojeab91801fcdd84
diff --git a/cmd/agentmesh-node/main.go b/cmd/agentmesh-node/main.go
index 351d7a9c..bf685be7 100644
--- a/cmd/agentmesh-node/main.go
+++ b/cmd/agentmesh-node/main.go
@@ -87,6 +87,9 @@ var (
allowLoopbackFlag bool
announcePrivateFlag bool
reachabilityFlag string
+ routersFlag int
+ routerSelectorFlags []string
+ routerPreferFlags []string
monitorBootstrapFlag time.Duration
monitorCheckIntervalFlag time.Duration
autoRelayMinIntervalFlag time.Duration
@@ -440,6 +443,15 @@ func main() {
}
}
+ routerSelector, err := api.ParseLabelFlags(routerSelectorFlags)
+ if err != nil {
+ logger.Fatalf("Invalid --router-selector: %v", err)
+ }
+ routerPrefer, err := api.ParseLabelFlags(routerPreferFlags)
+ if err != nil {
+ logger.Fatalf("Invalid --router-prefer: %v", err)
+ }
+
buildRuntimeNodeOptions := func(priv crypto.PrivKey, cpPub ed25519.PublicKey, rAddrs []multiaddr.Multiaddr) node.Options {
return node.Options{
PrivKey: priv,
@@ -455,6 +467,9 @@ func main() {
AllowLoopback: allowLoopbackFlag,
AnnouncePrivateAddrs: &announcePrivateFlag,
Reachability: node.Reachability(reachabilityFlag),
+ Routers: routersFlag,
+ RouterSelector: routerSelector,
+ RouterPrefer: routerPrefer,
MonitorBootstrap: monitorBootstrapFlag,
MonitorInterval: monitorCheckIntervalFlag,
AutoRelayMinInterval: autoRelayMinIntervalFlag,
@@ -865,6 +880,9 @@ func main() {
runCmd.Flags().BoolVar(&allowLoopbackFlag, "allow-loopback", false, "Allow publishing and connecting to loopback/link-local addresses")
runCmd.Flags().BoolVar(&announcePrivateFlag, "announce-private", true, "Publish this host's private (RFC1918/ULA) addresses to the mesh; keep enabled for LAN or on-premises meshes, disable when peers are only reachable through routers or public addresses")
runCmd.Flags().StringVar(&reachabilityFlag, "reachability", string(node.ReachabilityPrivate), "private: always hold a relay reservation and be reached through the routers, which works behind any NAT. auto: let AutoNAT v2 decide; a node the routers can dial on a public address advertises it and skips the relay. Use auto only where the routers are on the public side of every NAT this node is behind")
+ runCmd.Flags().IntVar(&routersFlag, "routers", node.DefaultRouters, "Routers to keep a session with, out of those the control plane lists (every router when there are fewer); one is enough to be on the mesh, the others are there for when it restarts")
+ runCmd.Flags().StringArrayVar(&routerSelectorFlags, "router-selector", nil, "Attach only to routers that attest this key=value label, repeatable, every pair required (e.g. --router-selector region=eu)")
+ runCmd.Flags().StringArrayVar(&routerPreferFlags, "router-prefer", nil, "Attach to routers that attest this key=value label before others, repeatable, the more pairs attested the earlier (e.g. --router-prefer zone=eu-a)")
runCmd.Flags().BoolVar(&offlineAccessFlag, "offline-access", false, "With --join, request OIDC offline access/refresh token for automatic renewal")
joinCmd.Flags().BoolVar(&allowLoopbackFlag, "allow-loopback", false, "Allow publishing and connecting to loopback/link-local addresses")
joinCmd.Flags().BoolVar(&announcePrivateFlag, "announce-private", true, "Publish this host's private (RFC1918/ULA) addresses to the mesh; keep enabled for LAN or on-premises meshes, disable when peers are only reachable through routers or public addresses")
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 3aa73e4b..dae889e7 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -1223,7 +1223,7 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {
finalRoles := []string{nodeRecord.Role}
finalRoles = append(finalRoles, customAccessRoles...)
- nodeRecord.Labels = filterAllowedLabels(allowedLabelPatterns(finalRoles, policyRoles), nodeRecord.Labels)
+ nodeRecord.Labels = labelsStillAllowed(canonical, allowedLabelPatterns(finalRoles, policyRoles), nodeRecord.Labels)
bBytes, _, err := identity.MintBiscuitToken(privKey, claims, nil, pID, biscuitExpiry, finalRoles, policyRoles, nodeRecord.Labels)
if err != nil {
@@ -1235,7 +1235,7 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {
} else {
// Bootstrap node
finalRoles, _ := nodeRoles(nodeRecord, bindings)
- nodeRecord.Labels = filterAllowedLabels(allowedLabelPatterns(finalRoles, policyRoles), nodeRecord.Labels)
+ nodeRecord.Labels = labelsStillAllowed(canonical, allowedLabelPatterns(finalRoles, policyRoles), nodeRecord.Labels)
bBytes, err := identity.MintBootstrapBiscuitToken(privKey, pID, nodeRecord.Role, biscuitExpiry, policyRoles, nodeRecord.Labels)
if err != nil {
logger.Errorf("Failed to mint refreshed token for node %s: %v", nodeRecord.PeerID, err)
@@ -3370,6 +3370,22 @@ func allowedLabelPatterns(roles []string, policyRoles []*api.PolicyRole) []strin
// filterAllowedLabels returns the subset of labels that are still permitted by
// patterns. On /refresh, labels declared at enrollment time whose grant has
// since been removed from the policy are dropped from the re-minted Biscuit.
+// labelsStillAllowed is filterAllowedLabels for a credential refresh, where
+// a label the policy no longer allows is dropped from the new credential
+// rather than ending the session. Nobody asked for the drop, so it is
+// logged: a router that loses the label a node's --router-selector names
+// stops being chosen, and the operator who narrowed allowed_labels should
+// see why.
+func labelsStillAllowed(peerID string, patterns []string, labels map[string]string) map[string]string {
+ kept := filterAllowedLabels(patterns, labels)
+ for k, v := range labels {
+ if kept[k] != v {
+ logger.Warnw("Label no longer allowed by policy, dropped at refresh", "peer_id", peerID, "label", k+"="+v)
+ }
+ }
+ return kept
+}
+
func filterAllowedLabels(patterns []string, labels map[string]string) map[string]string {
if len(labels) == 0 {
return nil
diff --git a/internal/node/connection_monitor.go b/internal/node/connection_monitor.go
index 276cd71b..8b87a734 100644
--- a/internal/node/connection_monitor.go
+++ b/internal/node/connection_monitor.go
@@ -14,100 +14,57 @@
package node
-import (
- "context"
-
- "github.com/multiformats/go-multiaddr"
-)
-
-type routerConnectionManager interface {
- IsConnected() bool
- LoadMeshConfig() ([]byte, []string, error)
- ConnectAndAuthWithRouter(ctx context.Context, addr multiaddr.Multiaddr) error
- LoadControlPlaneURL() (string, error)
-
- SaveMeshConfig(pubKey []byte, addrs []string) error
- UpdateRelays(addrs []multiaddr.Multiaddr)
+import "context"
+
+// routerAttacher is the node as the connection monitor sees it: how many
+// routers it keeps, how many it wants, and the two ways to get more.
+type routerAttacher interface {
+ // AttachedRouters is how many routers the node holds a live session with
+ // among those it keeps.
+ AttachedRouters() int
+ // WantRouters is how many it keeps: Options.Routers, or every router it
+ // knows of when there are fewer.
+ WantRouters() int
+ // TopUp attaches to more routers from what the node knows and returns how
+ // many it then keeps.
+ TopUp(ctx context.Context) int
+ // RefreshRouters re-reads the routers from the control plane.
+ RefreshRouters(ctx context.Context) error
}
-// checkRouterConnection monitors the connection to the routers and attempts to recover it if disconnected.
-// It returns two booleans indicating the current state of the connection:
-// - stable: true if the connection was already established and is healthy.
-// - reconnected: true if the connection was lost but successfully recovered during this check.
+// checkRouterConnection keeps the node attached to the routers it wants.
+// It returns two booleans:
+// - stable: the node holds at least one router and did not have to
+// recover one on this check. Short of its count but on the mesh, it
+// tops up and still reports stable: the mesh is there, the margin is
+// being restored.
+// - reconnected: the node had no router and found one, either among the
+// routers it already knew of or after re-reading them from the control
+// plane. Neither is a failure of the mesh.
//
-// The recovery process follows these steps:
-// 1. Connection Check: If already connected, return (stable=true, reconnected=false).
-// 2. P2P Retry: If disconnected, attempt to reconnect using the known P2P multiaddresses stored locally.
-// 3. HTTP Fallback: If P2P retries fail, fall back to HTTP discovery using the stored Control plane URL to fetch
-// the new router addresses and Peer IDs, update the local config, and attempt to reconnect.
-// 4. Total Failure: If all reconnection attempts fail, return (stable=false, reconnected=false).
-func checkRouterConnection(ctx context.Context, mgr routerConnectionManager) (stable bool, reconnected bool) {
- if mgr.IsConnected() {
- return true, false
- }
-
- logger.Warn("[Monitor] Disconnected from router. Attempting to reconnect...")
-
- var p2pAddrs []multiaddr.Multiaddr
- if _, storedAddrs, err := mgr.LoadMeshConfig(); err == nil {
- for _, addrStr := range storedAddrs {
- if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {
- p2pAddrs = append(p2pAddrs, ma)
- }
+// Only a node left with no router at all, after both, reports a failure;
+// the caller counts those and exits on too many in a row.
+func checkRouterConnection(ctx context.Context, mgr routerAttacher) (stable bool, reconnected bool) {
+ have, want := mgr.AttachedRouters(), mgr.WantRouters()
+ if have > 0 {
+ if have < want {
+ logger.Infof("[Monitor] Attached to %d of %d routers; attaching to more", have, want)
+ mgr.TopUp(ctx)
}
+ return true, false
}
- var connectedAddrs []multiaddr.Multiaddr
- for _, addr := range p2pAddrs {
- if err := mgr.ConnectAndAuthWithRouter(ctx, addr); err == nil {
- logger.Infof("[Monitor] Successfully reconnected to router via P2P: %s", addr)
- connectedAddrs = append(connectedAddrs, addr)
- }
- }
- if len(connectedAddrs) > 0 {
- mgr.UpdateRelays(connectedAddrs)
+ logger.Warn("[Monitor] Disconnected from every router. Attempting to reconnect...")
+ if mgr.TopUp(ctx) > 0 {
return false, true
}
-
- controlPlaneURL, err := mgr.LoadControlPlaneURL()
- if err != nil || controlPlaneURL == "" {
- return false, false
- }
-
- logger.Infof("[Monitor] Reconnect P2P failed. Discovering control plane info from %s...", controlPlaneURL)
- info, err := FetchControlPlaneInfo(ctx, controlPlaneURL)
- if err != nil || len(info.RouterAddresses) == 0 {
+ logger.Infof("[Monitor] No known router admitted this node. Re-reading the routers from the control plane...")
+ if err := mgr.RefreshRouters(ctx); err != nil {
+ logger.Warnf("[Monitor] Could not re-read the routers: %v", err)
return false, false
}
-
- var newRouterAddrs []multiaddr.Multiaddr
- for _, addrStr := range info.RouterAddresses {
- if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {
- newRouterAddrs = append(newRouterAddrs, ma)
- }
- }
-
- if len(newRouterAddrs) == 0 {
- return false, false
- }
-
- if pubKeyBytes, _, err := mgr.LoadMeshConfig(); err == nil {
- if saveErr := mgr.SaveMeshConfig(pubKeyBytes, info.RouterAddresses); saveErr != nil {
- logger.Errorf("[Monitor] Failed to save updated mesh config: %v", saveErr)
- }
- }
-
- var connectedAddrsFallback []multiaddr.Multiaddr
- for _, addr := range newRouterAddrs {
- if err := mgr.ConnectAndAuthWithRouter(ctx, addr); err == nil {
- logger.Infof("[Monitor] Successfully reconnected to router via HTTP fallback: %s", addr)
- connectedAddrsFallback = append(connectedAddrsFallback, addr)
- }
- }
- if len(connectedAddrsFallback) > 0 {
- mgr.UpdateRelays(connectedAddrsFallback)
+ if mgr.TopUp(ctx) > 0 {
return false, true
}
-
return false, false
}
diff --git a/internal/node/connection_monitor_test.go b/internal/node/connection_monitor_test.go
index ee9f5c16..317e320d 100644
--- a/internal/node/connection_monitor_test.go
+++ b/internal/node/connection_monitor_test.go
@@ -18,122 +18,80 @@ import (
"context"
"errors"
"testing"
-
- "github.com/google/agentmesh/api"
- "github.com/multiformats/go-multiaddr"
)
-type mockRouterConnectionManager struct {
- connected bool
- meshConfigErr error
- storedAddrs []string
- controlPlaneURLErr error
- controlPlaneURL string
- discoverErr error
- discoverResp *api.ControlPlaneInfoResponse
- connectP2PErr error
- connectHTTPReq bool
- connectHTTPErr error
- saveConfigErr error
-}
-
-func (m *mockRouterConnectionManager) IsConnected() bool {
- return m.connected
-}
-
-func (m *mockRouterConnectionManager) LoadMeshConfig() ([]byte, []string, error) {
- return nil, m.storedAddrs, m.meshConfigErr
+// fakeAttacher scripts a node for the monitor: how many routers it holds,
+// how many it wants, how many each top-up yields, and whether the control
+// plane answers.
+type fakeAttacher struct {
+ attached int
+ want int
+ topUps []int // what AttachedRouters reads after each TopUp, in order
+ refreshErr error
+ refreshed bool
+ topUpCalls int
}
-func (m *mockRouterConnectionManager) ConnectAndAuthWithRouter(ctx context.Context, addr multiaddr.Multiaddr) error {
- if m.connectHTTPReq {
- return m.connectHTTPErr
+func (f *fakeAttacher) AttachedRouters() int { return f.attached }
+func (f *fakeAttacher) WantRouters() int { return f.want }
+func (f *fakeAttacher) TopUp(context.Context) int {
+ if f.topUpCalls < len(f.topUps) {
+ f.attached = f.topUps[f.topUpCalls]
}
- return m.connectP2PErr
+ f.topUpCalls++
+ return f.attached
}
-
-func (m *mockRouterConnectionManager) LoadControlPlaneURL() (string, error) {
- return m.controlPlaneURL, m.controlPlaneURLErr
-}
-
-func (m *mockRouterConnectionManager) DiscoverControlPlaneInfo(ctx context.Context, url string) (*api.ControlPlaneInfoResponse, error) {
- return m.discoverResp, m.discoverErr
+func (f *fakeAttacher) RefreshRouters(context.Context) error {
+ f.refreshed = true
+ return f.refreshErr
}
-func (m *mockRouterConnectionManager) SaveMeshConfig(pubKey []byte, addrs []string) error {
- return m.saveConfigErr
-}
-
-func (m *mockRouterConnectionManager) UpdateRelays(addrs []multiaddr.Multiaddr) {}
-
func TestCheckRouterConnection(t *testing.T) {
cases := []struct {
- name string
- mgr *mockRouterConnectionManager
- wantStable bool
- wantReconn bool
+ name string
+ mgr *fakeAttacher
+ wantStable bool
+ wantReconn bool
+ wantTopUps int
+ wantRefreshed bool
}{
{
- name: "already connected",
- mgr: &mockRouterConnectionManager{
- connected: true,
- },
+ name: "holds every router it wants",
+ mgr: &fakeAttacher{attached: 2, want: 2},
wantStable: true,
- wantReconn: false,
},
{
- name: "disconnected, reconnects via P2P",
- mgr: &mockRouterConnectionManager{
- connected: false,
- storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},
- connectP2PErr: nil,
- },
- wantStable: false,
- wantReconn: true,
+ // On the mesh with one router short: the margin is restored,
+ // and this is not a failure of the mesh.
+ name: "short of its count tops up and stays stable",
+ mgr: &fakeAttacher{attached: 1, want: 2, topUps: []int{2}},
+ wantStable: true,
+ wantTopUps: 1,
},
{
- name: "disconnected, P2P fails, reconnects via HTTP",
- mgr: &mockRouterConnectionManager{
- connected: false,
- storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},
- connectP2PErr: errors.New("p2p failed"),
- controlPlaneURL: "https://cp.example.com",
- discoverResp: &api.ControlPlaneInfoResponse{
- RouterAddresses: []string{"/ip4/127.0.0.1/tcp/4002"},
- },
- connectHTTPReq: true,
- connectHTTPErr: nil,
- },
- wantStable: false,
+ name: "no router, reattaches from what it knows",
+ mgr: &fakeAttacher{attached: 0, want: 2, topUps: []int{1}},
wantReconn: true,
+ wantTopUps: 1,
},
{
- name: "disconnected, all fail",
- mgr: &mockRouterConnectionManager{
- connected: false,
- storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},
- connectP2PErr: errors.New("p2p failed"),
- controlPlaneURL: "https://cp.example.com",
- discoverResp: &api.ControlPlaneInfoResponse{
- RouterAddresses: []string{"/ip4/127.0.0.1/tcp/4002"},
- },
- connectHTTPReq: true,
- connectHTTPErr: errors.New("http fallback connect failed"),
- },
- wantStable: false,
- wantReconn: false,
+ name: "no router, reattaches after re-reading the control plane",
+ mgr: &fakeAttacher{attached: 0, want: 2, topUps: []int{0, 2}},
+ wantReconn: true,
+ wantTopUps: 2,
+ wantRefreshed: true,
},
{
- name: "disconnected, HTTP discovery fails",
- mgr: &mockRouterConnectionManager{
- connected: false,
- storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},
- connectP2PErr: errors.New("p2p failed"),
- controlPlaneURL: "https://cp.example.com",
- discoverErr: errors.New("discovery failed"),
- },
- wantStable: false,
- wantReconn: false,
+ name: "no router, control plane unreachable",
+ mgr: &fakeAttacher{attached: 0, want: 2, topUps: []int{0}, refreshErr: errors.New("dial tcp: connection refused")},
+ wantTopUps: 1,
+ wantRefreshed: true,
+ },
+ {
+ name: "no router admits it even after re-reading",
+ mgr: &fakeAttacher{attached: 0, want: 2, topUps: []int{0, 0}},
+ wantTopUps: 2,
+ wantRefreshed: true,
},
}
@@ -141,10 +99,16 @@ func TestCheckRouterConnection(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
stable, reconnected := checkRouterConnection(context.Background(), tc.mgr)
if stable != tc.wantStable {
- t.Errorf("expected stable %v, got %v", tc.wantStable, stable)
+ t.Errorf("stable = %v, want %v", stable, tc.wantStable)
}
if reconnected != tc.wantReconn {
- t.Errorf("expected reconnected %v, got %v", tc.wantReconn, reconnected)
+ t.Errorf("reconnected = %v, want %v", reconnected, tc.wantReconn)
+ }
+ if tc.mgr.topUpCalls != tc.wantTopUps {
+ t.Errorf("top-ups = %d, want %d", tc.mgr.topUpCalls, tc.wantTopUps)
+ }
+ if tc.mgr.refreshed != tc.wantRefreshed {
+ t.Errorf("refreshed = %v, want %v", tc.mgr.refreshed, tc.wantRefreshed)
}
})
}
diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.go
index 617ef98d..83b9eb94 100644
--- a/internal/node/controlplane_sync.go
+++ b/internal/node/controlplane_sync.go
@@ -193,7 +193,7 @@ func (n *AgentMeshNode) syncMeshInfo(ctx context.Context, controlPlaneURL string
if err != nil {
return err
}
- if len(info.RouterAddresses) > 0 {
+ if len(info.RouterAddresses) > 0 || len(info.GetRouters()) > 0 {
pubKey, _, loadErr := n.Store.LoadMeshConfig()
switch {
case loadErr != nil:
@@ -208,6 +208,9 @@ func (n *AgentMeshNode) syncMeshInfo(ctx context.Context, controlPlaneURL string
n.config.RouterAddrs = addrs
}
}
+ // What the node chooses its routers from, with the labels and load
+ // the stored addresses do not carry.
+ n.setRouterCatalog(candidatesFromInfo(ctx, info))
}
n.reconcileBannedPeers(info.GetBannedPeerIds(), fetchedAt)
return nil
diff --git a/internal/node/debug_handlers.go b/internal/node/debug_handlers.go
index 59224c87..ae546933 100644
--- a/internal/node/debug_handlers.go
+++ b/internal/node/debug_handlers.go
@@ -19,6 +19,7 @@ import (
"encoding/json"
"fmt"
"net/http"
+ "slices"
"time"
"github.com/libp2p/go-libp2p/core/peer"
@@ -114,11 +115,12 @@ func writeDebugJSON(w http.ResponseWriter, v any) {
// api/agentmesh.proto mesh contract.
type meshInfoResponse struct {
- PeerID string `json:"peer_id"`
- ConnectedPeers []string `json:"connected_peers"`
- DHTSize int `json:"dht_size"`
- RouterPeerID string `json:"router_peer_id"`
- LocalAPISocket string `json:"local_api_socket,omitempty"`
+ PeerID string `json:"peer_id"`
+ ConnectedPeers []string `json:"connected_peers"`
+ AttachedRouters []string `json:"attached_routers"`
+ DHTSize int `json:"dht_size"`
+ RouterPeerID string `json:"router_peer_id"`
+ LocalAPISocket string `json:"local_api_socket,omitempty"`
}
type connectivityResponse struct {
@@ -160,13 +162,22 @@ func (n *AgentMeshNode) meshInfo() (*meshInfoResponse, error) {
for _, p := range peers {
connectedPeers = append(connectedPeers, p.String())
}
+ // Routers the node holds as relays, a subset of the connected peers: a
+ // DHT query connects the node to other routers without attaching to them.
+ attached := n.attachedRouters()
+ attachedRouters := make([]string, 0, len(attached))
+ for _, p := range attached {
+ attachedRouters = append(attachedRouters, p.String())
+ }
+ slices.Sort(attachedRouters)
return &meshInfoResponse{
- PeerID: n.Host.ID().String(),
- ConnectedPeers: connectedPeers,
- DHTSize: n.DHT.RoutingTable().Size(),
- RouterPeerID: n.RouterPeerID.String(),
- LocalAPISocket: n.BoundSocketPath,
+ PeerID: n.Host.ID().String(),
+ ConnectedPeers: connectedPeers,
+ AttachedRouters: attachedRouters,
+ DHTSize: n.DHT.RoutingTable().Size(),
+ RouterPeerID: n.RouterPeerID.String(),
+ LocalAPISocket: n.BoundSocketPath,
}, nil
}
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index 78c1700a..c07d5ac8 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -31,7 +31,6 @@ import (
golog "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
- "github.com/multiformats/go-multiaddr"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -82,7 +81,7 @@ func (n *AgentMeshNode) Enroll(ctx context.Context, controlPlaneURL string, jwt
// Routers already holding a session are shown the new credential;
// connectToRouters skips them, as their sessions are live.
n.readmitRouters(ctx)
- return n.connectToRouters(ctx, enrollResp.RouterAddresses)
+ return n.connectToRouters(ctx, controlPlaneURL, enrollResp.RouterAddresses)
}
// adoptEnrolledKeys widens the trust set from the single key an enrollment
@@ -225,36 +224,37 @@ func (n *AgentMeshNode) processEnrollResponse(resp *http.Response) (*api.EnrollR
return &enrollResp, nil
}
-func (n *AgentMeshNode) connectToRouters(ctx context.Context, addrs []string) error {
+func (n *AgentMeshNode) connectToRouters(ctx context.Context, controlPlaneURL string, addrs []string) error {
if len(addrs) == 0 {
return fmt.Errorf("failed to connect and authenticate after HTTP enrollment: control plane returned no router addresses")
}
- if err := n.joinRouters(ctx, addrs); err != nil {
+ if err := n.joinRouters(ctx, controlPlaneURL, addrs); err != nil {
return fmt.Errorf("failed to connect and authenticate with any router after HTTP enrollment: %w", err)
}
logger.Info("Successfully enrolled via HTTP and stored identity and mesh config.")
return nil
}
-// joinRouters is the handshake that follows an enrollment: every router
-// the control plane answered with is dialled at once and the node is in
-// the mesh when one admits it, retried as at Start, since the routers are
-// busiest exactly when a fleet enrolls.
-func (n *AgentMeshNode) joinRouters(ctx context.Context, addrs []string) error {
- parsed := make([]multiaddr.Multiaddr, 0, len(addrs))
- for _, addrStr := range addrs {
- addr, err := multiaddr.NewMultiaddr(addrStr)
- if err != nil {
- logger.Warnf("Failed to parse router address from response: %v", err)
- continue
- }
- parsed = append(parsed, addr)
- }
- routers := routerPeers(ctx, parsed)
- if len(routers) == 0 {
+// joinRouters is the handshake that follows an enrollment: the node attaches
+// through the same admission as at Start, so the routers are chosen and
+// retried the same way. The enrollment answer lists addresses alone, so the
+// catalog is read from /info first, where the routers carry the labels a
+// selector matches; the answer is the fallback when /info cannot be read.
+func (n *AgentMeshNode) joinRouters(ctx context.Context, controlPlaneURL string, addrs []string) error {
+ var cands []routerCandidate
+ if info, err := FetchControlPlaneInfo(ctx, controlPlaneURL); err == nil {
+ cands = candidatesFromInfo(ctx, info)
+ } else {
+ logger.Debugf("Routers not read from /info after enrollment, using the enrollment answer: %v", err)
+ }
+ if len(cands) == 0 {
+ cands = candidatesFromAddrs(ctx, parseMultiaddrs(addrs))
+ }
+ if len(cands) == 0 {
return fmt.Errorf("no router peer behind %v", addrs)
}
- _, err := n.admitByRouters(ctx, routers)
+ n.setRouterCatalog(cands)
+ _, err := n.admitByRouters(ctx)
return err
}
@@ -426,7 +426,7 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL str
if len(enrollResp.RouterAddresses) == 0 {
return fmt.Errorf("failed to connect and authenticate after bootstrap enrollment: control plane returned no router addresses")
}
- if err := n.joinRouters(ctx, enrollResp.RouterAddresses); err != nil {
+ if err := n.joinRouters(ctx, controlPlaneURL, enrollResp.RouterAddresses); err != nil {
return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment: %w", err)
}
diff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.go
index 26b42e3f..6b00d065 100644
--- a/internal/node/enroll_test.go
+++ b/internal/node/enroll_test.go
@@ -764,7 +764,7 @@ func TestConnectToRouters_EmptyAddrs(t *testing.T) {
t.Fatal(err)
}
- err = node.connectToRouters(context.Background(), nil)
+ err = node.connectToRouters(context.Background(), "", nil)
if err == nil || !strings.Contains(err.Error(), "returned no router addresses") {
t.Fatalf("Expected no router addresses error, got: %v", err)
}
diff --git a/internal/node/node.go b/internal/node/node.go
index 97728608..ad12750f 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -171,7 +171,15 @@ type AgentMeshNode struct {
RouterPeerID peer.ID
authenticatedRouters map[peer.ID]bool
// redialing holds the routers a redialRouter goroutine is working on.
- redialing map[peer.ID]bool
+ redialing map[peer.ID]bool
+ // routerCatalog is every router the node knows of; attached are the
+ // Options.Routers of them it keeps a session with; shunned are the ones
+ // it stays off until the time given, having been sent away or having
+ // not come back. connMgr protects the attached ones from being trimmed.
+ routerCatalog []routerCandidate
+ attached map[peer.ID]bool
+ shunned map[peer.ID]time.Time
+ connMgr *connmgr.BasicConnMgr
peerLastEventTime map[string]int64
mu sync.Mutex
nodeConfig *NodeConfigComplete
@@ -532,18 +540,23 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
// Layer 2: Attach the Bouncer (Gater)
gater := &nodeConnGate{node: n}
- // The routers, one entry per peer with every address it is known by;
- // they double as the static relays.
- staticRelays := routerPeers(ctx, n.config.RouterAddrs)
+ // The routers, one entry per peer with every address it is known by.
+ // SyncControlPlane may already have listed them with labels and load;
+ // otherwise they are the stored addresses, which say nothing more.
+ if len(n.routerCandidates()) == 0 {
+ n.setRouterCatalog(candidatesFromAddrs(ctx, n.config.RouterAddrs))
+ }
+ staticRelays := n.routerCandidates()
if n.RouterPeerID == "" && len(staticRelays) > 0 {
n.RouterPeerID = staticRelays[0].ID
}
- logger.Infof("Configured %d static relays: %v", len(staticRelays), staticRelays)
+ logger.Infof("Configured %d routers, attaching to %d: %v", len(staticRelays), n.wantRouters(), staticRelays)
cm, err := connmgr.NewConnManager(100, 400, connmgr.WithGracePeriod(time.Minute))
if err != nil {
return fmt.Errorf("failed to create connection manager: %w", err)
}
+ n.connMgr = cm
// Layer 1: Transports & NAT Services. TLS first: Go peers and the
// Node/Python SDKs land on it. Noise is what a browser can speak, and a
@@ -574,7 +587,6 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
// If we have routers, configure them as our static fallback relays for NAT hole-punching
if len(staticRelays) > 0 {
- n.currentRelays = staticRelays
opts = append(opts, libp2p.EnableAutoRelayWithPeerSource(
func(ctx context.Context, numPeers int) <-chan peer.AddrInfo {
logger.Infof("[Relay] AutoRelay called PeerSource for %d peers", numPeers)
@@ -589,10 +601,15 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
case <-ctx.Done():
logger.Infof("[Relay] PeerSource context done")
case <-n.authSuccess:
- logger.Infof("[Relay] Yielding static relays to AutoRelay")
- // Shuffle the relays to distribute load evenly across routers
- shuffled := make([]peer.AddrInfo, len(currentRelays))
- copy(shuffled, currentRelays)
+ // The routers the node is attached to, which are the
+ // ones that will grant it a reservation; shuffled so
+ // a fleet spreads its reservations across them.
+ n.mu.Lock()
+ attachedRelays := n.currentRelays
+ n.mu.Unlock()
+ logger.Infof("[Relay] Yielding %d attached routers to AutoRelay", len(attachedRelays))
+ shuffled := make([]peer.AddrInfo, len(attachedRelays))
+ copy(shuffled, attachedRelays)
rand.Shuffle(len(shuffled), func(i, j int) {
shuffled[i], shuffled[j] = shuffled[j], shuffled[i]
})
@@ -603,6 +620,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
}()
return c
},
+ autorelay.WithNumRelays(n.config.Routers),
autorelay.WithBootDelay(n.config.AutoRelayBootDelay),
autorelay.WithBackoff(n.config.AutoRelayBackoff),
autorelay.WithMinInterval(n.config.AutoRelayMinInterval),
@@ -643,15 +661,18 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
n.mu.Lock()
_, wasAuthenticated := n.authenticatedRouters[remotePeer]
delete(n.authenticatedRouters, remotePeer)
+ attached := n.attached[remotePeer]
n.mu.Unlock()
if wasAuthenticated {
logger.Warnf("[AuthN] Router %s disconnected; clearing authenticated session so the next check re-handshakes", remotePeer)
- n.redialRouter(ctx, remotePeer)
+ if attached {
+ n.redialRouter(ctx, remotePeer)
+ }
}
},
})
- // Permanently add the static relay address to the peerstore so we can build relay paths later
+ // Permanently add the router addresses to the peerstore so we can build relay paths later
for _, pi := range staticRelays {
h.Peerstore().AddAddrs(pi.ID, pi.Addrs, peerstore.PermanentAddrTTL)
}
@@ -687,7 +708,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
n.services.reprovideNow = n.triggerReprovide
n.applyPendingEgress(ctx)
- authenticated, err := n.admitByRouters(ctx, staticRelays)
+ authenticated, err := n.admitByRouters(ctx)
if err != nil {
return err
}
@@ -993,23 +1014,44 @@ dhtLoop:
return nil
}
-// admitByRouters runs authWithRouters until a router admits this node, with
-// RouterRedialDelay doubling between attempts, up to RouterAuthAttempts of
-// them. A router that is up but slow, as every router is while a fleet
-// joins through it, fails the first handshake on its deadline and admits
-// the node on the next; without the retry a member that had just enrolled
-// exited at that first deadline. A router that refuses the credential ends
-// it at once, and no router at all is reported as before. authenticated is
-// false with a nil error only when there are no routers to admit the node.
-func (n *AgentMeshNode) admitByRouters(ctx context.Context, routers []peer.AddrInfo) (authenticated bool, err error) {
- if len(routers) == 0 {
+// admitByRouters attaches the node to the routers it keeps and returns as
+// soon as one has admitted it, retrying with RouterRedialDelay doubling
+// between attempts, up to RouterAuthAttempts of them. Each attempt chooses
+// afresh from the catalog, re-read from the control plane when it can be,
+// so a router that is up but slow, as every router is while a fleet joins
+// through it, admits the node on the next attempt, and a router that
+// restarts with the labels a selector names is found when it is back. A
+// router that refuses the credential ends it at once, and no router at all
+// is reported as before. authenticated is false with a nil error only when
+// the node knows of no router to admit it.
+func (n *AgentMeshNode) admitByRouters(ctx context.Context) (authenticated bool, err error) {
+ if len(n.routerCandidates()) == 0 {
return false, nil
}
+ biscuitBytes, err := n.loadIdentityForAuth()
+ if err != nil {
+ return false, fmt.Errorf("fatal auth failure: %w", err)
+ }
delay := n.config.RouterRedialDelay
for attempt := 1; ; attempt++ {
- authenticated, fatal := n.authWithRouters(ctx, routers)
- if authenticated {
- return true, nil
+ if attempt > 1 {
+ if refreshErr := n.refreshRouterCatalog(ctx); refreshErr != nil {
+ logger.Debugf("[AuthN] Routers not re-read from the control plane: %v", refreshErr)
+ }
+ }
+ targets := n.pickRouters(n.wantRouters())
+ var fatal error
+ if len(targets) > 0 {
+ results := n.dialAndAttach(ctx, targets, biscuitBytes)
+ for range targets {
+ err := <-results
+ if err == nil {
+ return true, nil
+ }
+ if errors.Is(err, ErrFatalAuth) {
+ fatal = err
+ }
+ }
}
if fatal != nil {
return false, fmt.Errorf("fatal auth failure: %w", fatal)
@@ -1027,34 +1069,6 @@ func (n *AgentMeshNode) admitByRouters(ctx context.Context, routers []peer.AddrI
}
}
-// authWithRouters dials every router at once and returns as soon as one has
-// admitted this node, or once all have refused. A router that is down costs
-// its dial timeout; paid up front and one router after another, that was
-// every member's cold-join cost for as long as the control plane still
-// listed a router that had gone. The dials still in flight finish in the
-// background and record their sessions as they land; the connection monitor
-// re-handshakes whatever drops later.
-func (n *AgentMeshNode) authWithRouters(ctx context.Context, routers []peer.AddrInfo) (authenticated bool, fatal error) {
- if len(routers) == 0 {
- return false, nil
- }
- biscuitBytes, err := n.loadIdentityForAuth()
- if err != nil {
- return false, err
- }
- results := n.dialRouters(ctx, routers, biscuitBytes)
- for range routers {
- err := <-results
- if err == nil {
- return true, nil
- }
- if errors.Is(err, ErrFatalAuth) {
- fatal = err
- }
- }
- return false, fatal
-}
-
// dialRouters runs authRouter against every router at once and delivers one
// outcome per router on the returned channel, in completion order. The
// channel is buffered for all of them, so a caller may stop reading early.
@@ -1116,9 +1130,13 @@ func (n *AgentMeshNode) redialRouter(ctx context.Context, router peer.ID) {
if n.isAuthenticatedAndConnected(router) {
return
}
+ if !n.isAttached(router) {
+ return
+ }
addrs := n.Host.Peerstore().Addrs(router)
if len(addrs) == 0 {
- logger.Warnf("[AuthN] Router %s dropped and no address for it is known; leaving it to the connection monitor", router)
+ logger.Warnf("[AuthN] Router %s dropped and no address for it is known; replacing it", router)
+ n.detach(router)
return
}
err := n.authRouter(ctx, peer.AddrInfo{ID: router, Addrs: addrs}, biscuitBytes)
@@ -1129,12 +1147,18 @@ func (n *AgentMeshNode) redialRouter(ctx context.Context, router peer.ID) {
}
if errors.Is(err, ErrFatalAuth) {
logger.Warnf("[AuthN] Router %s refused this node's credential on redial: %v", router, err)
+ n.detach(router)
+ n.shun(router, n.config.RouterShunDuration)
return
}
logger.Debugf("[AuthN] Redial %d of router %s failed: %v", attempt, router, err)
delay *= 2
}
- logger.Warnf("[AuthN] Router %s did not come back within %d redials; leaving it to the connection monitor", router, n.config.RouterRedialAttempts)
+ // Given up: the monitor tops up from the other routers, and this one
+ // is a candidate again once it is back on /info and the shun lapses.
+ logger.Warnf("[AuthN] Router %s did not come back within %d redials; replacing it", router, n.config.RouterRedialAttempts)
+ n.detach(router)
+ n.shun(router, n.config.RouterShunDuration)
}()
}
diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.go
index c53c20c1..1ebd7a4b 100644
--- a/internal/node/node_metrics.go
+++ b/internal/node/node_metrics.go
@@ -40,6 +40,8 @@ type nodeStateCollector struct {
dhtDesc *prometheus.Desc
biscuitExpiryDesc *prometheus.Desc
servicesDesc *prometheus.Desc
+ routersDesc *prometheus.Desc
+ routerCandsDesc *prometheus.Desc
}
func newNodeStateCollector(n *AgentMeshNode) *nodeStateCollector {
@@ -73,13 +75,21 @@ func newNodeStateCollector(n *AgentMeshNode) *nodeStateCollector {
"agentmesh_node_services_registered",
"Local services this node offers to the mesh, by type",
[]string{"type"}, nil),
+ routersDesc: prometheus.NewDesc(
+ "agentmesh_node_routers",
+ "Routers this node keeps a session with: attached now, and wanted (--routers, or every router listed when fewer)",
+ []string{"state"}, nil),
+ routerCandsDesc: prometheus.NewDesc(
+ "agentmesh_node_router_candidates",
+ "Routers the control plane lists that this node may attach to: all of them, and those attesting --router-selector",
+ []string{"match"}, nil),
}
}
func (c *nodeStateCollector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.readyDesc, c.meshConnectedDesc, c.connectedDesc, c.authenticatedDesc,
- c.dhtDesc, c.biscuitExpiryDesc, c.servicesDesc,
+ c.dhtDesc, c.biscuitExpiryDesc, c.servicesDesc, c.routersDesc, c.routerCandsDesc,
} {
ch <- d
}
@@ -100,6 +110,13 @@ func (c *nodeStateCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.meshConnectedDesc, prometheus.GaugeValue, connected)
ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(n.Host.Network().Peers())))
+ ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.AttachedRouters()), "attached")
+ ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.wantRouters()), "wanted")
+ cands := n.routerCandidates()
+ _, matching := selectRouters(cands, 0, n.config.RouterSelector, nil, nil)
+ ch <- prometheus.MustNewConstMetric(c.routerCandsDesc, prometheus.GaugeValue, float64(len(cands)), "any")
+ ch <- prometheus.MustNewConstMetric(c.routerCandsDesc, prometheus.GaugeValue, float64(matching), "selector")
+
authenticated := 0
n.authPeers.Range(func(_, _ any) bool {
authenticated++
diff --git a/internal/node/options.go b/internal/node/options.go
index 91d76e5d..f86d04aa 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -40,6 +40,15 @@ const (
// for a router that is busy admitting a fleet, short enough that a
// member with no router at all still reports it within a minute.
DefaultRouterAuthAttempts = 4
+ // DefaultRouters is how many routers a node keeps a session with: one
+ // is enough to be on the mesh, the second is there for when the first
+ // restarts. Every session costs a router a connection, so a node does
+ // not keep one with every router it knows of.
+ DefaultRouters = 2
+ // DefaultRouterShunDuration is how long a node stays off a router that
+ // sent it away or did not come back: past a rollout of the router, short
+ // enough that one back for good is used again.
+ DefaultRouterShunDuration = 5 * time.Minute
// DefaultSocketName is the local API socket the node creates in its data directory.
DefaultSocketName = "agentmesh.sock"
)
@@ -101,6 +110,23 @@ type Options struct {
// an enrollment, dial the routers before giving up, RouterRedialDelay
// doubling between attempts. Zero uses the default.
RouterAuthAttempts int
+ // Routers is how many routers the node keeps a session with, out of
+ // those the control plane lists; every router when there are fewer.
+ // Zero uses DefaultRouters. A session that drops is redialled, and a
+ // router that does not come back is replaced from the list.
+ Routers int
+ // RouterSelector names labels a router must attest for the node to
+ // attach to it, every pair; RouterPrefer names labels whose routers the
+ // node attaches to first, the more pairs attested the earlier. Among
+ // routers equal on preference the node takes the ones with the most
+ // room. Both are matched against the labels /info lists for a router,
+ // which are the ones the control plane attested at its enrollment.
+ RouterSelector map[string]string
+ RouterPrefer map[string]string
+ // RouterShunDuration is how long the node stays off a router that told
+ // it to go away or that did not come back on redial. Zero uses the
+ // default.
+ RouterShunDuration time.Duration
// BiscuitTimeout bounds Datalog evaluation when verifying biscuit tokens.
BiscuitTimeout time.Duration
// DHT Options
@@ -184,6 +210,12 @@ func (o *Options) Default() {
if o.RouterAuthAttempts <= 0 {
o.RouterAuthAttempts = DefaultRouterAuthAttempts
}
+ if o.Routers <= 0 {
+ o.Routers = DefaultRouters
+ }
+ if o.RouterShunDuration <= 0 {
+ o.RouterShunDuration = DefaultRouterShunDuration
+ }
if o.Reachability == "" {
o.Reachability = ReachabilityPrivate
}
@@ -245,6 +277,12 @@ func (o *Options) Validate() error {
default:
return fmt.Errorf("reachability %q is not %q or %q", o.Reachability, ReachabilityPrivate, ReachabilityAuto)
}
+ if err := api.ValidateLabels(o.RouterSelector); err != nil {
+ return fmt.Errorf("RouterSelector: %w", err)
+ }
+ if err := api.ValidateLabels(o.RouterPrefer); err != nil {
+ return fmt.Errorf("RouterPrefer: %w", err)
+ }
// ed25519 verification panics on a wrong-size key, and this one comes
// from a flag or FFI config.
if len(o.ControlPlanePubKey) > 0 && len(o.ControlPlanePubKey) != ed25519.PublicKeySize {
diff --git a/internal/node/routers.go b/internal/node/routers.go
new file mode 100644
index 00000000..e80015e8
--- /dev/null
+++ b/internal/node/routers.go
@@ -0,0 +1,349 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math/rand/v2"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/google/agentmesh/api"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/peerstore"
+ "github.com/multiformats/go-multiaddr"
+)
+
+// routerCandidate is one router as the node knows it: where to reach it,
+// the labels the control plane attested for it and its load as of its last
+// lease. From /info when the control plane answered, from stored addresses
+// alone when it did not.
+type routerCandidate struct {
+ peer.AddrInfo
+ Labels map[string]string
+ Connections int
+ ConnectionLimit int
+}
+
+// routerTag protects an attached router's connection from the node's own
+// connection manager, which otherwise trims by age and tag value and would
+// take the router before a stranger.
+const routerTag = "agentmesh-router"
+
+// candidatesFromInfo reads the routers /info lists. A control plane that
+// predates RouterInfo lists addresses only; those become candidates without
+// labels or load.
+func candidatesFromInfo(ctx context.Context, info *api.ControlPlaneInfoResponse) []routerCandidate {
+ if len(info.GetRouters()) == 0 {
+ return candidatesFromAddrs(ctx, parseMultiaddrs(info.GetRouterAddresses()))
+ }
+ var out []routerCandidate
+ for _, r := range info.GetRouters() {
+ for _, pi := range routerPeers(ctx, parseMultiaddrs(r.GetAddresses())) {
+ out = append(out, routerCandidate{
+ AddrInfo: pi,
+ Labels: r.GetLabels(),
+ Connections: int(r.GetConnections()),
+ ConnectionLimit: int(r.GetConnectionLimit()),
+ })
+ }
+ }
+ return out
+}
+
+func candidatesFromAddrs(ctx context.Context, addrs []multiaddr.Multiaddr) []routerCandidate {
+ peers := routerPeers(ctx, addrs)
+ out := make([]routerCandidate, 0, len(peers))
+ for _, pi := range peers {
+ out = append(out, routerCandidate{AddrInfo: pi})
+ }
+ return out
+}
+
+// selectRouters chooses up to k routers from cands: those that attest every
+// pair of selector and are not excluded, ordered by how many pairs of prefer
+// they attest, then by how full they are. A little noise on the load keeps a
+// fleet that reads the same /info from piling onto one router before its
+// next lease reports the change. The second result is how many candidates
+// passed the selector, so a caller can tell "none match" from "all taken".
+func selectRouters(cands []routerCandidate, k int, selector, prefer map[string]string, exclude map[peer.ID]bool) ([]peer.AddrInfo, int) {
+ type scored struct {
+ routerCandidate
+ preferred int
+ load float64
+ }
+ var eligible []scored
+ matched := 0
+ for _, c := range cands {
+ if len(selector) > 0 && !api.LabelsSatisfy(selector, c.Labels) {
+ continue
+ }
+ matched++
+ if exclude[c.ID] {
+ continue
+ }
+ preferred := 0
+ for key, value := range prefer {
+ if c.Labels[key] == value {
+ preferred++
+ }
+ }
+ load := 0.5
+ if c.ConnectionLimit > 0 {
+ load = float64(c.Connections) / float64(c.ConnectionLimit)
+ }
+ load += rand.Float64() * 0.05
+ eligible = append(eligible, scored{c, preferred, load})
+ }
+ sort.SliceStable(eligible, func(i, j int) bool {
+ if eligible[i].preferred != eligible[j].preferred {
+ return eligible[i].preferred > eligible[j].preferred
+ }
+ return eligible[i].load < eligible[j].load
+ })
+ if k > len(eligible) {
+ k = len(eligible)
+ }
+ out := make([]peer.AddrInfo, 0, k)
+ for _, s := range eligible[:k] {
+ out = append(out, s.AddrInfo)
+ }
+ return out, matched
+}
+
+// setRouterCatalog replaces what the node knows about the routers and keeps
+// their addresses in the peerstore, so a circuit through any of them can be
+// built later.
+func (n *AgentMeshNode) setRouterCatalog(cands []routerCandidate) {
+ n.mu.Lock()
+ n.routerCatalog = cands
+ n.mu.Unlock()
+ if n.Host != nil {
+ for _, c := range cands {
+ n.Host.Peerstore().AddAddrs(c.ID, c.Addrs, peerstore.PermanentAddrTTL)
+ }
+ }
+}
+
+func (n *AgentMeshNode) routerCandidates() []routerCandidate {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ return append([]routerCandidate(nil), n.routerCatalog...)
+}
+
+// refreshRouterCatalog re-reads /info. Best effort: a node started from its
+// stored addresses with no control plane in reach keeps what it has.
+func (n *AgentMeshNode) refreshRouterCatalog(ctx context.Context) error {
+ controlPlaneURL, err := n.Store.LoadControlPlaneURL()
+ if err != nil || controlPlaneURL == "" {
+ return errors.New("no control plane URL in store")
+ }
+ info, err := FetchControlPlaneInfo(ctx, controlPlaneURL)
+ if err != nil {
+ return err
+ }
+ if len(info.GetRouterAddresses()) == 0 && len(info.GetRouters()) == 0 {
+ return errors.New("control plane lists no routers")
+ }
+ n.setRouterCatalog(candidatesFromInfo(ctx, info))
+ return nil
+}
+
+// wantRouters is how many routers the node holds: its --routers, capped by
+// the routers that attest its selector, so a selector that one router
+// matches is a node that holds one router and is not short.
+func (n *AgentMeshNode) wantRouters() int {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ matched := 0
+ for _, c := range n.routerCatalog {
+ if len(n.config.RouterSelector) == 0 || api.LabelsSatisfy(n.config.RouterSelector, c.Labels) {
+ matched++
+ }
+ }
+ return min(n.config.Routers, matched)
+}
+
+// attachedRouters lists the routers the node keeps and holds a live
+// authenticated session with.
+func (n *AgentMeshNode) attachedRouters() []peer.ID {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ var out []peer.ID
+ for pid := range n.attached {
+ if n.authenticatedRouters[pid] && n.Host.Network().Connectedness(pid) == network.Connected {
+ out = append(out, pid)
+ }
+ }
+ return out
+}
+
+// AttachedRouters is how many routers the node keeps a live session with.
+func (n *AgentMeshNode) AttachedRouters() int { return len(n.attachedRouters()) }
+
+// WantRouters is how many routers the node keeps.
+func (n *AgentMeshNode) WantRouters() int { return n.wantRouters() }
+
+// RefreshRouters re-reads the routers from the control plane.
+func (n *AgentMeshNode) RefreshRouters(ctx context.Context) error { return n.refreshRouterCatalog(ctx) }
+
+// attach marks a router as one the node keeps: protected from the node's
+// connection manager, redialled when it drops, and offered to AutoRelay for
+// a reservation.
+func (n *AgentMeshNode) attach(router peer.ID) {
+ n.mu.Lock()
+ if n.attached == nil {
+ n.attached = map[peer.ID]bool{}
+ }
+ n.attached[router] = true
+ n.mu.Unlock()
+ if n.connMgr != nil {
+ n.connMgr.Protect(router, routerTag)
+ }
+ n.refreshRelays()
+}
+
+// detach gives a router up: the node stops redialling it and lets the
+// monitor top up from the others.
+func (n *AgentMeshNode) detach(router peer.ID) {
+ n.mu.Lock()
+ delete(n.attached, router)
+ n.mu.Unlock()
+ if n.connMgr != nil {
+ n.connMgr.Unprotect(router, routerTag)
+ }
+ n.refreshRelays()
+}
+
+func (n *AgentMeshNode) isAttached(router peer.ID) bool {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ return n.attached[router]
+}
+
+// refreshRelays offers AutoRelay the routers the node is attached to: a
+// reservation needs an authenticated session, which only those have.
+func (n *AgentMeshNode) refreshRelays() {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ var relays []peer.AddrInfo
+ for _, c := range n.routerCatalog {
+ if n.attached[c.ID] {
+ relays = append(relays, c.AddrInfo)
+ }
+ }
+ n.currentRelays = relays
+}
+
+// pickRouters chooses k routers to attach to, from the catalog, leaving out
+// those already attached and those that sent the node away. When the
+// catalog has routers and none passes the selector, the warning names both
+// sides, since that is a configuration to fix rather than a router to wait
+// for.
+func (n *AgentMeshNode) pickRouters(k int) []peer.AddrInfo {
+ cands := n.routerCandidates()
+ n.mu.Lock()
+ exclude := make(map[peer.ID]bool, len(n.attached)+len(n.shunned))
+ for pid := range n.attached {
+ exclude[pid] = true
+ }
+ now := time.Now()
+ for pid, until := range n.shunned {
+ if now.Before(until) {
+ exclude[pid] = true
+ } else {
+ delete(n.shunned, pid)
+ }
+ }
+ n.mu.Unlock()
+ picked, matched := selectRouters(cands, k, n.config.RouterSelector, n.config.RouterPrefer, exclude)
+ if matched == 0 && len(cands) > 0 && len(n.config.RouterSelector) > 0 {
+ var seen []string
+ for _, c := range cands {
+ seen = append(seen, fmt.Sprintf("%s %v", c.ID.ShortString(), c.Labels))
+ }
+ logger.Warnf("[AuthN] No router attests the selector %v; routers listed: %s", n.config.RouterSelector, strings.Join(seen, ", "))
+ }
+ return picked
+}
+
+// shun keeps the node off a router for a while: one that told it to go
+// away, or one that would not come back on redial.
+func (n *AgentMeshNode) shun(router peer.ID, d time.Duration) {
+ n.mu.Lock()
+ if n.shunned == nil {
+ n.shunned = map[peer.ID]time.Time{}
+ }
+ n.shunned[router] = time.Now().Add(d)
+ n.mu.Unlock()
+}
+
+// dialAndAttach runs authRouter against each router at once and attaches
+// the ones that admit the node, delivering one outcome per router on the
+// returned channel in completion order. A caller may stop reading early:
+// the dials still in flight attach on their own as they land.
+func (n *AgentMeshNode) dialAndAttach(ctx context.Context, routers []peer.AddrInfo, biscuitBytes []byte) <-chan error {
+ results := make(chan error, len(routers))
+ for _, router := range routers {
+ go func() {
+ err := n.authRouter(ctx, router, biscuitBytes)
+ if err != nil {
+ logger.Warnf("[AuthN] Failed to bootstrap and auth with router %s: %v", router.ID, err)
+ } else {
+ n.attach(router.ID)
+ }
+ results <- err
+ }()
+ }
+ return results
+}
+
+// topUpRouters attaches to routers until the node keeps as many as it
+// wants, choosing from the catalog, and returns how many it then keeps.
+// Every dial is waited for here: this is the monitor's path, where there
+// is no one to be ready for.
+func (n *AgentMeshNode) topUpRouters(ctx context.Context) int {
+ have := n.AttachedRouters()
+ want := n.wantRouters()
+ if have >= want {
+ return have
+ }
+ biscuitBytes, err := n.loadIdentityForAuth()
+ if err != nil {
+ logger.Errorf("[AuthN] Cannot attach to routers without an identity: %v", err)
+ return have
+ }
+ targets := n.pickRouters(want - have)
+ if len(targets) == 0 {
+ return have
+ }
+ results := n.dialAndAttach(ctx, targets, biscuitBytes)
+ for range targets {
+ <-results
+ }
+ after := n.AttachedRouters()
+ if after > have {
+ n.triggerReprovide()
+ }
+ return after
+}
+
+// TopUp attaches to more routers from what the node knows and returns how
+// many it then keeps.
+func (n *AgentMeshNode) TopUp(ctx context.Context) int { return n.topUpRouters(ctx) }
diff --git a/internal/node/routers_test.go b/internal/node/routers_test.go
new file mode 100644
index 00000000..38a8e5ab
--- /dev/null
+++ b/internal/node/routers_test.go
@@ -0,0 +1,166 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "testing"
+
+ "github.com/google/agentmesh/api"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/multiformats/go-multiaddr"
+)
+
+func candidate(t *testing.T, name string, labels map[string]string, connections, limit int) routerCandidate {
+ t.Helper()
+ // A peer ID is derived from the name so the test can tell them apart.
+ id, err := peer.Decode("12D3KooW" + name)
+ if err != nil {
+ // Not a real ID: build one from the name's bytes instead.
+ id = peer.ID("router-" + name)
+ }
+ addr := multiaddr.StringCast("/ip4/127.0.0.1/tcp/4501")
+ return routerCandidate{
+ AddrInfo: peer.AddrInfo{ID: id, Addrs: []multiaddr.Multiaddr{addr}},
+ Labels: labels,
+ Connections: connections,
+ ConnectionLimit: limit,
+ }
+}
+
+func ids(infos []peer.AddrInfo) []peer.ID {
+ out := make([]peer.ID, 0, len(infos))
+ for _, pi := range infos {
+ out = append(out, pi.ID)
+ }
+ return out
+}
+
+// TestSelectRouters pins the choice of routers: the selector is a
+// requirement on every pair, the preference orders by pairs attested, load
+// orders among equals, and routers already attached or shunned are left
+// out. The second result tells "none match" from "all taken".
+func TestSelectRouters(t *testing.T) {
+ euEmpty := candidate(t, "eu-empty", map[string]string{"region": "eu", "zone": "eu-a"}, 100, 4000)
+ euFull := candidate(t, "eu-full", map[string]string{"region": "eu", "zone": "eu-b"}, 3900, 4000)
+ usEmpty := candidate(t, "us-empty", map[string]string{"region": "us"}, 0, 4000)
+ unlabelled := candidate(t, "plain", nil, 0, 0)
+ all := []routerCandidate{euFull, usEmpty, unlabelled, euEmpty}
+
+ t.Run("by load when nothing else is said", func(t *testing.T) {
+ picked, matched := selectRouters(all, 2, nil, nil, nil)
+ if matched != 4 {
+ t.Fatalf("matched = %d, want all 4", matched)
+ }
+ got := ids(picked)
+ // The full router comes last; the unlabelled one with unknown load
+ // sits in the middle at 0.5.
+ if len(got) != 2 || got[0] == euFull.ID || got[1] == euFull.ID {
+ t.Fatalf("picked %v, want the two with the most room", got)
+ }
+ })
+
+ t.Run("selector is a requirement on every pair", func(t *testing.T) {
+ picked, matched := selectRouters(all, 3, map[string]string{"region": "eu", "zone": "eu-a"}, nil, nil)
+ if matched != 1 || len(picked) != 1 || picked[0].ID != euEmpty.ID {
+ t.Fatalf("picked %v (matched %d), want only eu-a", ids(picked), matched)
+ }
+ _, matched = selectRouters(all, 3, map[string]string{"region": "apac"}, nil, nil)
+ if matched != 0 {
+ t.Fatalf("matched = %d for a region no router attests, want 0", matched)
+ }
+ })
+
+ t.Run("preference orders before load", func(t *testing.T) {
+ picked, _ := selectRouters(all, 2, nil, map[string]string{"region": "eu"}, nil)
+ got := ids(picked)
+ if len(got) != 2 || got[0] != euEmpty.ID || got[1] != euFull.ID {
+ t.Fatalf("picked %v, want the eu routers first, the emptier one before the full one", got)
+ }
+ // More pairs attested ranks higher.
+ picked, _ = selectRouters(all, 1, nil, map[string]string{"region": "eu", "zone": "eu-b"}, nil)
+ if len(picked) != 1 || picked[0].ID != euFull.ID {
+ t.Fatalf("picked %v, want eu-b for attesting both preferred pairs although full", ids(picked))
+ }
+ })
+
+ t.Run("attached and shunned routers are left out, but still count as matching", func(t *testing.T) {
+ exclude := map[peer.ID]bool{euEmpty.ID: true, usEmpty.ID: true}
+ picked, matched := selectRouters(all, 5, nil, nil, exclude)
+ if matched != 4 || len(picked) != 2 {
+ t.Fatalf("picked %d (matched %d), want 2 of 4", len(picked), matched)
+ }
+ for _, pi := range picked {
+ if exclude[pi.ID] {
+ t.Fatalf("picked an excluded router %s", pi.ID)
+ }
+ }
+ })
+
+ t.Run("k is clamped", func(t *testing.T) {
+ picked, _ := selectRouters(all[:1], 3, nil, nil, nil)
+ if len(picked) != 1 {
+ t.Fatalf("picked %d from 1 candidate, want 1", len(picked))
+ }
+ })
+}
+
+// A control plane that predates RouterInfo lists addresses alone; they
+// become candidates with no labels and unknown load, so a selector matches
+// none and a plain node still has something to attach to.
+func TestCandidatesFromInfoFallsBackToAddresses(t *testing.T) {
+ id, _ := peer.Decode("12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM")
+ addr := "/ip4/127.0.0.1/tcp/4501/p2p/" + id.String()
+ old := &api.ControlPlaneInfoResponse{RouterAddresses: []string{addr}}
+ cands := candidatesFromInfo(context.Background(), old)
+ if len(cands) != 1 || cands[0].ID != id || cands[0].Labels != nil || cands[0].ConnectionLimit != 0 {
+ t.Fatalf("candidates from an address-only /info = %+v, want one unlabelled router", cands)
+ }
+
+ current := &api.ControlPlaneInfoResponse{
+ RouterAddresses: []string{addr},
+ Routers: []*api.RouterInfo{{
+ PeerId: id.String(), Addresses: []string{addr},
+ Labels: map[string]string{"region": "eu"}, Connections: 10, ConnectionLimit: 4000,
+ }},
+ }
+ cands = candidatesFromInfo(context.Background(), current)
+ if len(cands) != 1 || cands[0].Labels["region"] != "eu" || cands[0].Connections != 10 {
+ t.Fatalf("candidates from /info = %+v, want the labelled router with its load", cands)
+ }
+}
+
+// A node wants as many routers as it is configured for, but no more than
+// attest its selector: with one matching router it is complete at one, and
+// the monitor does not keep topping up.
+func TestWantRoutersCountsSelectorMatches(t *testing.T) {
+ n := &AgentMeshNode{config: Options{Routers: 2}}
+ n.routerCatalog = []routerCandidate{
+ candidate(t, "a", map[string]string{"site": "a"}, 0, 100),
+ candidate(t, "b", map[string]string{"site": "b"}, 0, 100),
+ candidate(t, "c", nil, 0, 100),
+ }
+ if got := n.wantRouters(); got != 2 {
+ t.Fatalf("wantRouters without selector = %d, want 2", got)
+ }
+ n.config.RouterSelector = map[string]string{"site": "a"}
+ if got := n.wantRouters(); got != 1 {
+ t.Fatalf("wantRouters with selector site=a = %d, want 1", got)
+ }
+ n.config.RouterSelector = map[string]string{"site": "z"}
+ if got := n.wantRouters(); got != 0 {
+ t.Fatalf("wantRouters with a selector no router attests = %d, want 0", got)
+ }
+}
diff --git a/site/content/docs/reference/agentmesh-node.md b/site/content/docs/reference/agentmesh-node.md
index 619104cb..2347d777 100644
--- a/site/content/docs/reference/agentmesh-node.md
+++ b/site/content/docs/reference/agentmesh-node.md
@@ -110,6 +110,9 @@ explains how to enroll.
| `--listen` | `/ip4/0.0.0.0/udp/5001/quic-v1`, `/ip4/0.0.0.0/tcp/5002` | libp2p listen addresses. Repeatable. |
| `--allow-loopback`, `--announce-private`, `--router-connect-timeout` | | As for `join`. |
| `--reachability` | `private` | How the node decides whether peers can dial it directly. `private` always holds a relay reservation with the routers and is reached through them, which works behind any NAT. `auto` lets AutoNAT decide from dial-backs by the routers: a node they reach on a public address advertises it and holds no reservation, so direct dials replace relayed ones. Use `auto` only where the routers are on the public side of every NAT the node is behind; a router in the same cluster or LAN can reach a node the rest of the mesh cannot, and would make it unreachable. |
+| `--routers` | `2` | Routers to keep a session with, out of those the control plane lists; every router when there are fewer. One is enough to be on the mesh, the second is there for when the first restarts. A session that drops is redialled at 2 s, 4 s and so on; a router that does not come back is replaced from the list. A call reaches a peer on any router, attached or not. |
+| `--router-selector` | none | Attach only to routers whose credential attests this `key=value` label, repeatable, every pair required: `--router-selector region=eu`. The labels are the ones the control plane lists on `/info`, attested at the router's enrollment. A selector no router attests leaves the node retrying as if no router answered, with a warning naming the selector and the labels it saw. |
+| `--router-prefer` | none | Attach to routers attesting this `key=value` label before others, repeatable; a router attesting more of the pairs comes earlier, and among equals the one with the most room. `--router-prefer zone=eu-a` keeps a node on its zone's routers while they have capacity and lets it fall to the others. |
| `--enable-relay` | `false` | Relay traffic for other peers. |
| `--mesh` | `public-mesh` | Mesh federation name. Must match the routers' name. |
| `--discovery-interval` | `30s` | DHT discovery polling interval. |