Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: attach to two routers, chosen by label and load
Antonio Ojeab91801fcdd84
diff --git a/cmd/agentmesh-node/main.go b/cmd/agentmesh-node/main.goindex 351d7a9c..bf685be7 100644--- a/cmd/agentmesh-node/main.go+++ b/cmd/agentmesh-node/main.go@@ -87,6 +87,9 @@ var (allowLoopbackFlag boolannouncePrivateFlag boolreachabilityFlag string+ routersFlag int+ routerSelectorFlags []string+ routerPreferFlags []stringmonitorBootstrapFlag time.DurationmonitorCheckIntervalFlag time.DurationautoRelayMinIntervalFlag time.Duration@@ -440,6 +443,15 @@ func main() {}}+ routerSelector, err := api.ParseLabelFlags(routerSelectorFlags)+ if err != nil {+ logger.Fatalf("Invalid --router-selector: %v", err)+ }+ routerPrefer, err := api.ParseLabelFlags(routerPreferFlags)+ if err != nil {+ logger.Fatalf("Invalid --router-prefer: %v", err)+ }+buildRuntimeNodeOptions := func(priv crypto.PrivKey, cpPub ed25519.PublicKey, rAddrs []multiaddr.Multiaddr) node.Options {return node.Options{PrivKey: priv,@@ -455,6 +467,9 @@ func main() {AllowLoopback: allowLoopbackFlag,AnnouncePrivateAddrs: &announcePrivateFlag,Reachability: node.Reachability(reachabilityFlag),+ Routers: routersFlag,+ RouterSelector: routerSelector,+ RouterPrefer: routerPrefer,MonitorBootstrap: monitorBootstrapFlag,MonitorInterval: monitorCheckIntervalFlag,AutoRelayMinInterval: autoRelayMinIntervalFlag,@@ -865,6 +880,9 @@ func main() {runCmd.Flags().BoolVar(&allowLoopbackFlag, "allow-loopback", false, "Allow publishing and connecting to loopback/link-local addresses")runCmd.Flags().BoolVar(&announcePrivateFlag, "announce-private", true, "Publish this host's private (RFC1918/ULA) addresses to the mesh; keep enabled for LAN or on-premises meshes, disable when peers are only reachable through routers or public addresses")runCmd.Flags().StringVar(&reachabilityFlag, "reachability", string(node.ReachabilityPrivate), "private: always hold a relay reservation and be reached through the routers, which works behind any NAT. auto: let AutoNAT v2 decide; a node the routers can dial on a public address advertises it and skips the relay. Use auto only where the routers are on the public side of every NAT this node is behind")+ runCmd.Flags().IntVar(&routersFlag, "routers", node.DefaultRouters, "Routers to keep a session with, out of those the control plane lists (every router when there are fewer); one is enough to be on the mesh, the others are there for when it restarts")+ runCmd.Flags().StringArrayVar(&routerSelectorFlags, "router-selector", nil, "Attach only to routers that attest this key=value label, repeatable, every pair required (e.g. --router-selector region=eu)")+ runCmd.Flags().StringArrayVar(&routerPreferFlags, "router-prefer", nil, "Attach to routers that attest this key=value label before others, repeatable, the more pairs attested the earlier (e.g. --router-prefer zone=eu-a)")runCmd.Flags().BoolVar(&offlineAccessFlag, "offline-access", false, "With --join, request OIDC offline access/refresh token for automatic renewal")joinCmd.Flags().BoolVar(&allowLoopbackFlag, "allow-loopback", false, "Allow publishing and connecting to loopback/link-local addresses")joinCmd.Flags().BoolVar(&announcePrivateFlag, "announce-private", true, "Publish this host's private (RFC1918/ULA) addresses to the mesh; keep enabled for LAN or on-premises meshes, disable when peers are only reachable through routers or public addresses")diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex 3aa73e4b..dae889e7 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -1223,7 +1223,7 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {finalRoles := []string{nodeRecord.Role}finalRoles = append(finalRoles, customAccessRoles...)- nodeRecord.Labels = filterAllowedLabels(allowedLabelPatterns(finalRoles, policyRoles), nodeRecord.Labels)+ nodeRecord.Labels = labelsStillAllowed(canonical, allowedLabelPatterns(finalRoles, policyRoles), nodeRecord.Labels)bBytes, _, err := identity.MintBiscuitToken(privKey, claims, nil, pID, biscuitExpiry, finalRoles, policyRoles, nodeRecord.Labels)if err != nil {@@ -1235,7 +1235,7 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {} else {// Bootstrap nodefinalRoles, _ := nodeRoles(nodeRecord, bindings)- nodeRecord.Labels = filterAllowedLabels(allowedLabelPatterns(finalRoles, policyRoles), nodeRecord.Labels)+ nodeRecord.Labels = labelsStillAllowed(canonical, allowedLabelPatterns(finalRoles, policyRoles), nodeRecord.Labels)bBytes, err := identity.MintBootstrapBiscuitToken(privKey, pID, nodeRecord.Role, biscuitExpiry, policyRoles, nodeRecord.Labels)if err != nil {logger.Errorf("Failed to mint refreshed token for node %s: %v", nodeRecord.PeerID, err)@@ -3370,6 +3370,22 @@ func allowedLabelPatterns(roles []string, policyRoles []*api.PolicyRole) []strin// filterAllowedLabels returns the subset of labels that are still permitted by// patterns. On /refresh, labels declared at enrollment time whose grant has// since been removed from the policy are dropped from the re-minted Biscuit.+// labelsStillAllowed is filterAllowedLabels for a credential refresh, where+// a label the policy no longer allows is dropped from the new credential+// rather than ending the session. Nobody asked for the drop, so it is+// logged: a router that loses the label a node's --router-selector names+// stops being chosen, and the operator who narrowed allowed_labels should+// see why.+func labelsStillAllowed(peerID string, patterns []string, labels map[string]string) map[string]string {+ kept := filterAllowedLabels(patterns, labels)+ for k, v := range labels {+ if kept[k] != v {+ logger.Warnw("Label no longer allowed by policy, dropped at refresh", "peer_id", peerID, "label", k+"="+v)+ }+ }+ return kept+}+func filterAllowedLabels(patterns []string, labels map[string]string) map[string]string {if len(labels) == 0 {return nildiff --git a/internal/node/connection_monitor.go b/internal/node/connection_monitor.goindex 276cd71b..8b87a734 100644--- a/internal/node/connection_monitor.go+++ b/internal/node/connection_monitor.go@@ -14,100 +14,57 @@package node-import (- "context"-- "github.com/multiformats/go-multiaddr"-)--type routerConnectionManager interface {- IsConnected() bool- LoadMeshConfig() ([]byte, []string, error)- ConnectAndAuthWithRouter(ctx context.Context, addr multiaddr.Multiaddr) error- LoadControlPlaneURL() (string, error)-- SaveMeshConfig(pubKey []byte, addrs []string) error- UpdateRelays(addrs []multiaddr.Multiaddr)+import "context"++// routerAttacher is the node as the connection monitor sees it: how many+// routers it keeps, how many it wants, and the two ways to get more.+type routerAttacher interface {+ // AttachedRouters is how many routers the node holds a live session with+ // among those it keeps.+ AttachedRouters() int+ // WantRouters is how many it keeps: Options.Routers, or every router it+ // knows of when there are fewer.+ WantRouters() int+ // TopUp attaches to more routers from what the node knows and returns how+ // many it then keeps.+ TopUp(ctx context.Context) int+ // RefreshRouters re-reads the routers from the control plane.+ RefreshRouters(ctx context.Context) error}-// checkRouterConnection monitors the connection to the routers and attempts to recover it if disconnected.-// It returns two booleans indicating the current state of the connection:-// - stable: true if the connection was already established and is healthy.-// - reconnected: true if the connection was lost but successfully recovered during this check.+// checkRouterConnection keeps the node attached to the routers it wants.+// It returns two booleans:+// - stable: the node holds at least one router and did not have to+// recover one on this check. Short of its count but on the mesh, it+// tops up and still reports stable: the mesh is there, the margin is+// being restored.+// - reconnected: the node had no router and found one, either among the+// routers it already knew of or after re-reading them from the control+// plane. Neither is a failure of the mesh.//-// The recovery process follows these steps:-// 1. Connection Check: If already connected, return (stable=true, reconnected=false).-// 2. P2P Retry: If disconnected, attempt to reconnect using the known P2P multiaddresses stored locally.-// 3. HTTP Fallback: If P2P retries fail, fall back to HTTP discovery using the stored Control plane URL to fetch-// the new router addresses and Peer IDs, update the local config, and attempt to reconnect.-// 4. Total Failure: If all reconnection attempts fail, return (stable=false, reconnected=false).-func checkRouterConnection(ctx context.Context, mgr routerConnectionManager) (stable bool, reconnected bool) {- if mgr.IsConnected() {- return true, false- }-- logger.Warn("[Monitor] Disconnected from router. Attempting to reconnect...")-- var p2pAddrs []multiaddr.Multiaddr- if _, storedAddrs, err := mgr.LoadMeshConfig(); err == nil {- for _, addrStr := range storedAddrs {- if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {- p2pAddrs = append(p2pAddrs, ma)- }+// Only a node left with no router at all, after both, reports a failure;+// the caller counts those and exits on too many in a row.+func checkRouterConnection(ctx context.Context, mgr routerAttacher) (stable bool, reconnected bool) {+ have, want := mgr.AttachedRouters(), mgr.WantRouters()+ if have > 0 {+ if have < want {+ logger.Infof("[Monitor] Attached to %d of %d routers; attaching to more", have, want)+ mgr.TopUp(ctx)}+ return true, false}- var connectedAddrs []multiaddr.Multiaddr- for _, addr := range p2pAddrs {- if err := mgr.ConnectAndAuthWithRouter(ctx, addr); err == nil {- logger.Infof("[Monitor] Successfully reconnected to router via P2P: %s", addr)- connectedAddrs = append(connectedAddrs, addr)- }- }- if len(connectedAddrs) > 0 {- mgr.UpdateRelays(connectedAddrs)+ logger.Warn("[Monitor] Disconnected from every router. Attempting to reconnect...")+ if mgr.TopUp(ctx) > 0 {return false, true}-- controlPlaneURL, err := mgr.LoadControlPlaneURL()- if err != nil || controlPlaneURL == "" {- return false, false- }-- logger.Infof("[Monitor] Reconnect P2P failed. Discovering control plane info from %s...", controlPlaneURL)- info, err := FetchControlPlaneInfo(ctx, controlPlaneURL)- if err != nil || len(info.RouterAddresses) == 0 {+ logger.Infof("[Monitor] No known router admitted this node. Re-reading the routers from the control plane...")+ if err := mgr.RefreshRouters(ctx); err != nil {+ logger.Warnf("[Monitor] Could not re-read the routers: %v", err)return false, false}-- var newRouterAddrs []multiaddr.Multiaddr- for _, addrStr := range info.RouterAddresses {- if ma, err := multiaddr.NewMultiaddr(addrStr); err == nil {- newRouterAddrs = append(newRouterAddrs, ma)- }- }-- if len(newRouterAddrs) == 0 {- return false, false- }-- if pubKeyBytes, _, err := mgr.LoadMeshConfig(); err == nil {- if saveErr := mgr.SaveMeshConfig(pubKeyBytes, info.RouterAddresses); saveErr != nil {- logger.Errorf("[Monitor] Failed to save updated mesh config: %v", saveErr)- }- }-- var connectedAddrsFallback []multiaddr.Multiaddr- for _, addr := range newRouterAddrs {- if err := mgr.ConnectAndAuthWithRouter(ctx, addr); err == nil {- logger.Infof("[Monitor] Successfully reconnected to router via HTTP fallback: %s", addr)- connectedAddrsFallback = append(connectedAddrsFallback, addr)- }- }- if len(connectedAddrsFallback) > 0 {- mgr.UpdateRelays(connectedAddrsFallback)+ if mgr.TopUp(ctx) > 0 {return false, true}-return false, false}diff --git a/internal/node/connection_monitor_test.go b/internal/node/connection_monitor_test.goindex ee9f5c16..317e320d 100644--- a/internal/node/connection_monitor_test.go+++ b/internal/node/connection_monitor_test.go@@ -18,122 +18,80 @@ import ("context""errors""testing"-- "github.com/google/agentmesh/api"- "github.com/multiformats/go-multiaddr")-type mockRouterConnectionManager struct {- connected bool- meshConfigErr error- storedAddrs []string- controlPlaneURLErr error- controlPlaneURL string- discoverErr error- discoverResp *api.ControlPlaneInfoResponse- connectP2PErr error- connectHTTPReq bool- connectHTTPErr error- saveConfigErr error-}--func (m *mockRouterConnectionManager) IsConnected() bool {- return m.connected-}--func (m *mockRouterConnectionManager) LoadMeshConfig() ([]byte, []string, error) {- return nil, m.storedAddrs, m.meshConfigErr+// fakeAttacher scripts a node for the monitor: how many routers it holds,+// how many it wants, how many each top-up yields, and whether the control+// plane answers.+type fakeAttacher struct {+ attached int+ want int+ topUps []int // what AttachedRouters reads after each TopUp, in order+ refreshErr error+ refreshed bool+ topUpCalls int}-func (m *mockRouterConnectionManager) ConnectAndAuthWithRouter(ctx context.Context, addr multiaddr.Multiaddr) error {- if m.connectHTTPReq {- return m.connectHTTPErr+func (f *fakeAttacher) AttachedRouters() int { return f.attached }+func (f *fakeAttacher) WantRouters() int { return f.want }+func (f *fakeAttacher) TopUp(context.Context) int {+ if f.topUpCalls < len(f.topUps) {+ f.attached = f.topUps[f.topUpCalls]}- return m.connectP2PErr+ f.topUpCalls+++ return f.attached}--func (m *mockRouterConnectionManager) LoadControlPlaneURL() (string, error) {- return m.controlPlaneURL, m.controlPlaneURLErr-}--func (m *mockRouterConnectionManager) DiscoverControlPlaneInfo(ctx context.Context, url string) (*api.ControlPlaneInfoResponse, error) {- return m.discoverResp, m.discoverErr+func (f *fakeAttacher) RefreshRouters(context.Context) error {+ f.refreshed = true+ return f.refreshErr}-func (m *mockRouterConnectionManager) SaveMeshConfig(pubKey []byte, addrs []string) error {- return m.saveConfigErr-}--func (m *mockRouterConnectionManager) UpdateRelays(addrs []multiaddr.Multiaddr) {}-func TestCheckRouterConnection(t *testing.T) {cases := []struct {- name string- mgr *mockRouterConnectionManager- wantStable bool- wantReconn bool+ name string+ mgr *fakeAttacher+ wantStable bool+ wantReconn bool+ wantTopUps int+ wantRefreshed bool}{{- name: "already connected",- mgr: &mockRouterConnectionManager{- connected: true,- },+ name: "holds every router it wants",+ mgr: &fakeAttacher{attached: 2, want: 2},wantStable: true,- wantReconn: false,},{- name: "disconnected, reconnects via P2P",- mgr: &mockRouterConnectionManager{- connected: false,- storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},- connectP2PErr: nil,- },- wantStable: false,- wantReconn: true,+ // On the mesh with one router short: the margin is restored,+ // and this is not a failure of the mesh.+ name: "short of its count tops up and stays stable",+ mgr: &fakeAttacher{attached: 1, want: 2, topUps: []int{2}},+ wantStable: true,+ wantTopUps: 1,},{- name: "disconnected, P2P fails, reconnects via HTTP",- mgr: &mockRouterConnectionManager{- connected: false,- storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},- connectP2PErr: errors.New("p2p failed"),- controlPlaneURL: "https://cp.example.com",- discoverResp: &api.ControlPlaneInfoResponse{- RouterAddresses: []string{"/ip4/127.0.0.1/tcp/4002"},- },- connectHTTPReq: true,- connectHTTPErr: nil,- },- wantStable: false,+ name: "no router, reattaches from what it knows",+ mgr: &fakeAttacher{attached: 0, want: 2, topUps: []int{1}},wantReconn: true,+ wantTopUps: 1,},{- name: "disconnected, all fail",- mgr: &mockRouterConnectionManager{- connected: false,- storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},- connectP2PErr: errors.New("p2p failed"),- controlPlaneURL: "https://cp.example.com",- discoverResp: &api.ControlPlaneInfoResponse{- RouterAddresses: []string{"/ip4/127.0.0.1/tcp/4002"},- },- connectHTTPReq: true,- connectHTTPErr: errors.New("http fallback connect failed"),- },- wantStable: false,- wantReconn: false,+ name: "no router, reattaches after re-reading the control plane",+ mgr: &fakeAttacher{attached: 0, want: 2, topUps: []int{0, 2}},+ wantReconn: true,+ wantTopUps: 2,+ wantRefreshed: true,},{- name: "disconnected, HTTP discovery fails",- mgr: &mockRouterConnectionManager{- connected: false,- storedAddrs: []string{"/ip4/127.0.0.1/tcp/4001"},- connectP2PErr: errors.New("p2p failed"),- controlPlaneURL: "https://cp.example.com",- discoverErr: errors.New("discovery failed"),- },- wantStable: false,- wantReconn: false,+ name: "no router, control plane unreachable",+ mgr: &fakeAttacher{attached: 0, want: 2, topUps: []int{0}, refreshErr: errors.New("dial tcp: connection refused")},+ wantTopUps: 1,+ wantRefreshed: true,+ },+ {+ name: "no router admits it even after re-reading",+ mgr: &fakeAttacher{attached: 0, want: 2, topUps: []int{0, 0}},+ wantTopUps: 2,+ wantRefreshed: true,},}@@ -141,10 +99,16 @@ func TestCheckRouterConnection(t *testing.T) {t.Run(tc.name, func(t *testing.T) {stable, reconnected := checkRouterConnection(context.Background(), tc.mgr)if stable != tc.wantStable {- t.Errorf("expected stable %v, got %v", tc.wantStable, stable)+ t.Errorf("stable = %v, want %v", stable, tc.wantStable)}if reconnected != tc.wantReconn {- t.Errorf("expected reconnected %v, got %v", tc.wantReconn, reconnected)+ t.Errorf("reconnected = %v, want %v", reconnected, tc.wantReconn)+ }+ if tc.mgr.topUpCalls != tc.wantTopUps {+ t.Errorf("top-ups = %d, want %d", tc.mgr.topUpCalls, tc.wantTopUps)+ }+ if tc.mgr.refreshed != tc.wantRefreshed {+ t.Errorf("refreshed = %v, want %v", tc.mgr.refreshed, tc.wantRefreshed)}})}diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.goindex 617ef98d..83b9eb94 100644--- a/internal/node/controlplane_sync.go+++ b/internal/node/controlplane_sync.go@@ -193,7 +193,7 @@ func (n *AgentMeshNode) syncMeshInfo(ctx context.Context, controlPlaneURL stringif err != nil {return err}- if len(info.RouterAddresses) > 0 {+ if len(info.RouterAddresses) > 0 || len(info.GetRouters()) > 0 {pubKey, _, loadErr := n.Store.LoadMeshConfig()switch {case loadErr != nil:@@ -208,6 +208,9 @@ func (n *AgentMeshNode) syncMeshInfo(ctx context.Context, controlPlaneURL stringn.config.RouterAddrs = addrs}}+ // What the node chooses its routers from, with the labels and load+ // the stored addresses do not carry.+ n.setRouterCatalog(candidatesFromInfo(ctx, info))}n.reconcileBannedPeers(info.GetBannedPeerIds(), fetchedAt)return nildiff --git a/internal/node/debug_handlers.go b/internal/node/debug_handlers.goindex 59224c87..ae546933 100644--- a/internal/node/debug_handlers.go+++ b/internal/node/debug_handlers.go@@ -19,6 +19,7 @@ import ("encoding/json""fmt""net/http"+ "slices""time""github.com/libp2p/go-libp2p/core/peer"@@ -114,11 +115,12 @@ func writeDebugJSON(w http.ResponseWriter, v any) {// api/agentmesh.proto mesh contract.type meshInfoResponse struct {- PeerID string `json:"peer_id"`- ConnectedPeers []string `json:"connected_peers"`- DHTSize int `json:"dht_size"`- RouterPeerID string `json:"router_peer_id"`- LocalAPISocket string `json:"local_api_socket,omitempty"`+ PeerID string `json:"peer_id"`+ ConnectedPeers []string `json:"connected_peers"`+ AttachedRouters []string `json:"attached_routers"`+ DHTSize int `json:"dht_size"`+ RouterPeerID string `json:"router_peer_id"`+ LocalAPISocket string `json:"local_api_socket,omitempty"`}type connectivityResponse struct {@@ -160,13 +162,22 @@ func (n *AgentMeshNode) meshInfo() (*meshInfoResponse, error) {for _, p := range peers {connectedPeers = append(connectedPeers, p.String())}+ // Routers the node holds as relays, a subset of the connected peers: a+ // DHT query connects the node to other routers without attaching to them.+ attached := n.attachedRouters()+ attachedRouters := make([]string, 0, len(attached))+ for _, p := range attached {+ attachedRouters = append(attachedRouters, p.String())+ }+ slices.Sort(attachedRouters)return &meshInfoResponse{- PeerID: n.Host.ID().String(),- ConnectedPeers: connectedPeers,- DHTSize: n.DHT.RoutingTable().Size(),- RouterPeerID: n.RouterPeerID.String(),- LocalAPISocket: n.BoundSocketPath,+ PeerID: n.Host.ID().String(),+ ConnectedPeers: connectedPeers,+ AttachedRouters: attachedRouters,+ DHTSize: n.DHT.RoutingTable().Size(),+ RouterPeerID: n.RouterPeerID.String(),+ LocalAPISocket: n.BoundSocketPath,}, nil}diff --git a/internal/node/enroll.go b/internal/node/enroll.goindex 78c1700a..c07d5ac8 100644--- a/internal/node/enroll.go+++ b/internal/node/enroll.go@@ -31,7 +31,6 @@ import (golog "github.com/ipfs/go-log/v2""github.com/libp2p/go-libp2p/core/crypto""github.com/libp2p/go-libp2p/core/peer"- "github.com/multiformats/go-multiaddr""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/timestamppb")@@ -82,7 +81,7 @@ func (n *AgentMeshNode) Enroll(ctx context.Context, controlPlaneURL string, jwt// Routers already holding a session are shown the new credential;// connectToRouters skips them, as their sessions are live.n.readmitRouters(ctx)- return n.connectToRouters(ctx, enrollResp.RouterAddresses)+ return n.connectToRouters(ctx, controlPlaneURL, enrollResp.RouterAddresses)}// adoptEnrolledKeys widens the trust set from the single key an enrollment@@ -225,36 +224,37 @@ func (n *AgentMeshNode) processEnrollResponse(resp *http.Response) (*api.EnrollRreturn &enrollResp, nil}-func (n *AgentMeshNode) connectToRouters(ctx context.Context, addrs []string) error {+func (n *AgentMeshNode) connectToRouters(ctx context.Context, controlPlaneURL string, addrs []string) error {if len(addrs) == 0 {return fmt.Errorf("failed to connect and authenticate after HTTP enrollment: control plane returned no router addresses")}- if err := n.joinRouters(ctx, addrs); err != nil {+ if err := n.joinRouters(ctx, controlPlaneURL, addrs); err != nil {return fmt.Errorf("failed to connect and authenticate with any router after HTTP enrollment: %w", err)}logger.Info("Successfully enrolled via HTTP and stored identity and mesh config.")return nil}-// joinRouters is the handshake that follows an enrollment: every router-// the control plane answered with is dialled at once and the node is in-// the mesh when one admits it, retried as at Start, since the routers are-// busiest exactly when a fleet enrolls.-func (n *AgentMeshNode) joinRouters(ctx context.Context, addrs []string) error {- parsed := make([]multiaddr.Multiaddr, 0, len(addrs))- for _, addrStr := range addrs {- addr, err := multiaddr.NewMultiaddr(addrStr)- if err != nil {- logger.Warnf("Failed to parse router address from response: %v", err)- continue- }- parsed = append(parsed, addr)- }- routers := routerPeers(ctx, parsed)- if len(routers) == 0 {+// joinRouters is the handshake that follows an enrollment: the node attaches+// through the same admission as at Start, so the routers are chosen and+// retried the same way. The enrollment answer lists addresses alone, so the+// catalog is read from /info first, where the routers carry the labels a+// selector matches; the answer is the fallback when /info cannot be read.+func (n *AgentMeshNode) joinRouters(ctx context.Context, controlPlaneURL string, addrs []string) error {+ var cands []routerCandidate+ if info, err := FetchControlPlaneInfo(ctx, controlPlaneURL); err == nil {+ cands = candidatesFromInfo(ctx, info)+ } else {+ logger.Debugf("Routers not read from /info after enrollment, using the enrollment answer: %v", err)+ }+ if len(cands) == 0 {+ cands = candidatesFromAddrs(ctx, parseMultiaddrs(addrs))+ }+ if len(cands) == 0 {return fmt.Errorf("no router peer behind %v", addrs)}- _, err := n.admitByRouters(ctx, routers)+ n.setRouterCatalog(cands)+ _, err := n.admitByRouters(ctx)return err}@@ -426,7 +426,7 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL strif len(enrollResp.RouterAddresses) == 0 {return fmt.Errorf("failed to connect and authenticate after bootstrap enrollment: control plane returned no router addresses")}- if err := n.joinRouters(ctx, enrollResp.RouterAddresses); err != nil {+ if err := n.joinRouters(ctx, controlPlaneURL, enrollResp.RouterAddresses); err != nil {return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment: %w", err)}diff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.goindex 26b42e3f..6b00d065 100644--- a/internal/node/enroll_test.go+++ b/internal/node/enroll_test.go@@ -764,7 +764,7 @@ func TestConnectToRouters_EmptyAddrs(t *testing.T) {t.Fatal(err)}- err = node.connectToRouters(context.Background(), nil)+ err = node.connectToRouters(context.Background(), "", nil)if err == nil || !strings.Contains(err.Error(), "returned no router addresses") {t.Fatalf("Expected no router addresses error, got: %v", err)}diff --git a/internal/node/node.go b/internal/node/node.goindex 97728608..ad12750f 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -171,7 +171,15 @@ type AgentMeshNode struct {RouterPeerID peer.IDauthenticatedRouters map[peer.ID]bool// redialing holds the routers a redialRouter goroutine is working on.- redialing map[peer.ID]bool+ redialing map[peer.ID]bool+ // routerCatalog is every router the node knows of; attached are the+ // Options.Routers of them it keeps a session with; shunned are the ones+ // it stays off until the time given, having been sent away or having+ // not come back. connMgr protects the attached ones from being trimmed.+ routerCatalog []routerCandidate+ attached map[peer.ID]bool+ shunned map[peer.ID]time.Time+ connMgr *connmgr.BasicConnMgrpeerLastEventTime map[string]int64mu sync.MutexnodeConfig *NodeConfigComplete@@ -532,18 +540,23 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {// Layer 2: Attach the Bouncer (Gater)gater := &nodeConnGate{node: n}- // The routers, one entry per peer with every address it is known by;- // they double as the static relays.- staticRelays := routerPeers(ctx, n.config.RouterAddrs)+ // The routers, one entry per peer with every address it is known by.+ // SyncControlPlane may already have listed them with labels and load;+ // otherwise they are the stored addresses, which say nothing more.+ if len(n.routerCandidates()) == 0 {+ n.setRouterCatalog(candidatesFromAddrs(ctx, n.config.RouterAddrs))+ }+ staticRelays := n.routerCandidates()if n.RouterPeerID == "" && len(staticRelays) > 0 {n.RouterPeerID = staticRelays[0].ID}- logger.Infof("Configured %d static relays: %v", len(staticRelays), staticRelays)+ logger.Infof("Configured %d routers, attaching to %d: %v", len(staticRelays), n.wantRouters(), staticRelays)cm, err := connmgr.NewConnManager(100, 400, connmgr.WithGracePeriod(time.Minute))if err != nil {return fmt.Errorf("failed to create connection manager: %w", err)}+ n.connMgr = cm// Layer 1: Transports & NAT Services. TLS first: Go peers and the// Node/Python SDKs land on it. Noise is what a browser can speak, and a@@ -574,7 +587,6 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {// If we have routers, configure them as our static fallback relays for NAT hole-punchingif len(staticRelays) > 0 {- n.currentRelays = staticRelaysopts = append(opts, libp2p.EnableAutoRelayWithPeerSource(func(ctx context.Context, numPeers int) <-chan peer.AddrInfo {logger.Infof("[Relay] AutoRelay called PeerSource for %d peers", numPeers)@@ -589,10 +601,15 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {case <-ctx.Done():logger.Infof("[Relay] PeerSource context done")case <-n.authSuccess:- logger.Infof("[Relay] Yielding static relays to AutoRelay")- // Shuffle the relays to distribute load evenly across routers- shuffled := make([]peer.AddrInfo, len(currentRelays))- copy(shuffled, currentRelays)+ // The routers the node is attached to, which are the+ // ones that will grant it a reservation; shuffled so+ // a fleet spreads its reservations across them.+ n.mu.Lock()+ attachedRelays := n.currentRelays+ n.mu.Unlock()+ logger.Infof("[Relay] Yielding %d attached routers to AutoRelay", len(attachedRelays))+ shuffled := make([]peer.AddrInfo, len(attachedRelays))+ copy(shuffled, attachedRelays)rand.Shuffle(len(shuffled), func(i, j int) {shuffled[i], shuffled[j] = shuffled[j], shuffled[i]})@@ -603,6 +620,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {}()return c},+ autorelay.WithNumRelays(n.config.Routers),autorelay.WithBootDelay(n.config.AutoRelayBootDelay),autorelay.WithBackoff(n.config.AutoRelayBackoff),autorelay.WithMinInterval(n.config.AutoRelayMinInterval),@@ -643,15 +661,18 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {n.mu.Lock()_, wasAuthenticated := n.authenticatedRouters[remotePeer]delete(n.authenticatedRouters, remotePeer)+ attached := n.attached[remotePeer]n.mu.Unlock()if wasAuthenticated {logger.Warnf("[AuthN] Router %s disconnected; clearing authenticated session so the next check re-handshakes", remotePeer)- n.redialRouter(ctx, remotePeer)+ if attached {+ n.redialRouter(ctx, remotePeer)+ }}},})- // Permanently add the static relay address to the peerstore so we can build relay paths later+ // Permanently add the router addresses to the peerstore so we can build relay paths laterfor _, pi := range staticRelays {h.Peerstore().AddAddrs(pi.ID, pi.Addrs, peerstore.PermanentAddrTTL)}@@ -687,7 +708,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {n.services.reprovideNow = n.triggerReproviden.applyPendingEgress(ctx)- authenticated, err := n.admitByRouters(ctx, staticRelays)+ authenticated, err := n.admitByRouters(ctx)if err != nil {return err}@@ -993,23 +1014,44 @@ dhtLoop:return nil}-// admitByRouters runs authWithRouters until a router admits this node, with-// RouterRedialDelay doubling between attempts, up to RouterAuthAttempts of-// them. A router that is up but slow, as every router is while a fleet-// joins through it, fails the first handshake on its deadline and admits-// the node on the next; without the retry a member that had just enrolled-// exited at that first deadline. A router that refuses the credential ends-// it at once, and no router at all is reported as before. authenticated is-// false with a nil error only when there are no routers to admit the node.-func (n *AgentMeshNode) admitByRouters(ctx context.Context, routers []peer.AddrInfo) (authenticated bool, err error) {- if len(routers) == 0 {+// admitByRouters attaches the node to the routers it keeps and returns as+// soon as one has admitted it, retrying with RouterRedialDelay doubling+// between attempts, up to RouterAuthAttempts of them. Each attempt chooses+// afresh from the catalog, re-read from the control plane when it can be,+// so a router that is up but slow, as every router is while a fleet joins+// through it, admits the node on the next attempt, and a router that+// restarts with the labels a selector names is found when it is back. A+// router that refuses the credential ends it at once, and no router at all+// is reported as before. authenticated is false with a nil error only when+// the node knows of no router to admit it.+func (n *AgentMeshNode) admitByRouters(ctx context.Context) (authenticated bool, err error) {+ if len(n.routerCandidates()) == 0 {return false, nil}+ biscuitBytes, err := n.loadIdentityForAuth()+ if err != nil {+ return false, fmt.Errorf("fatal auth failure: %w", err)+ }delay := n.config.RouterRedialDelayfor attempt := 1; ; attempt++ {- authenticated, fatal := n.authWithRouters(ctx, routers)- if authenticated {- return true, nil+ if attempt > 1 {+ if refreshErr := n.refreshRouterCatalog(ctx); refreshErr != nil {+ logger.Debugf("[AuthN] Routers not re-read from the control plane: %v", refreshErr)+ }+ }+ targets := n.pickRouters(n.wantRouters())+ var fatal error+ if len(targets) > 0 {+ results := n.dialAndAttach(ctx, targets, biscuitBytes)+ for range targets {+ err := <-results+ if err == nil {+ return true, nil+ }+ if errors.Is(err, ErrFatalAuth) {+ fatal = err+ }+ }}if fatal != nil {return false, fmt.Errorf("fatal auth failure: %w", fatal)@@ -1027,34 +1069,6 @@ func (n *AgentMeshNode) admitByRouters(ctx context.Context, routers []peer.AddrI}}-// authWithRouters dials every router at once and returns as soon as one has-// admitted this node, or once all have refused. A router that is down costs-// its dial timeout; paid up front and one router after another, that was-// every member's cold-join cost for as long as the control plane still-// listed a router that had gone. The dials still in flight finish in the-// background and record their sessions as they land; the connection monitor-// re-handshakes whatever drops later.-func (n *AgentMeshNode) authWithRouters(ctx context.Context, routers []peer.AddrInfo) (authenticated bool, fatal error) {- if len(routers) == 0 {- return false, nil- }- biscuitBytes, err := n.loadIdentityForAuth()- if err != nil {- return false, err- }- results := n.dialRouters(ctx, routers, biscuitBytes)- for range routers {- err := <-results- if err == nil {- return true, nil- }- if errors.Is(err, ErrFatalAuth) {- fatal = err- }- }- return false, fatal-}-// dialRouters runs authRouter against every router at once and delivers one// outcome per router on the returned channel, in completion order. The// channel is buffered for all of them, so a caller may stop reading early.@@ -1116,9 +1130,13 @@ func (n *AgentMeshNode) redialRouter(ctx context.Context, router peer.ID) {if n.isAuthenticatedAndConnected(router) {return}+ if !n.isAttached(router) {+ return+ }addrs := n.Host.Peerstore().Addrs(router)if len(addrs) == 0 {- logger.Warnf("[AuthN] Router %s dropped and no address for it is known; leaving it to the connection monitor", router)+ logger.Warnf("[AuthN] Router %s dropped and no address for it is known; replacing it", router)+ n.detach(router)return}err := n.authRouter(ctx, peer.AddrInfo{ID: router, Addrs: addrs}, biscuitBytes)@@ -1129,12 +1147,18 @@ func (n *AgentMeshNode) redialRouter(ctx context.Context, router peer.ID) {}if errors.Is(err, ErrFatalAuth) {logger.Warnf("[AuthN] Router %s refused this node's credential on redial: %v", router, err)+ n.detach(router)+ n.shun(router, n.config.RouterShunDuration)return}logger.Debugf("[AuthN] Redial %d of router %s failed: %v", attempt, router, err)delay *= 2}- logger.Warnf("[AuthN] Router %s did not come back within %d redials; leaving it to the connection monitor", router, n.config.RouterRedialAttempts)+ // Given up: the monitor tops up from the other routers, and this one+ // is a candidate again once it is back on /info and the shun lapses.+ logger.Warnf("[AuthN] Router %s did not come back within %d redials; replacing it", router, n.config.RouterRedialAttempts)+ n.detach(router)+ n.shun(router, n.config.RouterShunDuration)}()}diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.goindex c53c20c1..1ebd7a4b 100644--- a/internal/node/node_metrics.go+++ b/internal/node/node_metrics.go@@ -40,6 +40,8 @@ type nodeStateCollector struct {dhtDesc *prometheus.DescbiscuitExpiryDesc *prometheus.DescservicesDesc *prometheus.Desc+ routersDesc *prometheus.Desc+ routerCandsDesc *prometheus.Desc}func newNodeStateCollector(n *AgentMeshNode) *nodeStateCollector {@@ -73,13 +75,21 @@ func newNodeStateCollector(n *AgentMeshNode) *nodeStateCollector {"agentmesh_node_services_registered","Local services this node offers to the mesh, by type",[]string{"type"}, nil),+ routersDesc: prometheus.NewDesc(+ "agentmesh_node_routers",+ "Routers this node keeps a session with: attached now, and wanted (--routers, or every router listed when fewer)",+ []string{"state"}, nil),+ routerCandsDesc: prometheus.NewDesc(+ "agentmesh_node_router_candidates",+ "Routers the control plane lists that this node may attach to: all of them, and those attesting --router-selector",+ []string{"match"}, nil),}}func (c *nodeStateCollector) Describe(ch chan<- *prometheus.Desc) {for _, d := range []*prometheus.Desc{c.readyDesc, c.meshConnectedDesc, c.connectedDesc, c.authenticatedDesc,- c.dhtDesc, c.biscuitExpiryDesc, c.servicesDesc,+ c.dhtDesc, c.biscuitExpiryDesc, c.servicesDesc, c.routersDesc, c.routerCandsDesc,} {ch <- d}@@ -100,6 +110,13 @@ func (c *nodeStateCollector) Collect(ch chan<- prometheus.Metric) {ch <- prometheus.MustNewConstMetric(c.meshConnectedDesc, prometheus.GaugeValue, connected)ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(n.Host.Network().Peers())))+ ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.AttachedRouters()), "attached")+ ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.wantRouters()), "wanted")+ cands := n.routerCandidates()+ _, matching := selectRouters(cands, 0, n.config.RouterSelector, nil, nil)+ ch <- prometheus.MustNewConstMetric(c.routerCandsDesc, prometheus.GaugeValue, float64(len(cands)), "any")+ ch <- prometheus.MustNewConstMetric(c.routerCandsDesc, prometheus.GaugeValue, float64(matching), "selector")+authenticated := 0n.authPeers.Range(func(_, _ any) bool {authenticated++diff --git a/internal/node/options.go b/internal/node/options.goindex 91d76e5d..f86d04aa 100644--- a/internal/node/options.go+++ b/internal/node/options.go@@ -40,6 +40,15 @@ const (// for a router that is busy admitting a fleet, short enough that a// member with no router at all still reports it within a minute.DefaultRouterAuthAttempts = 4+ // DefaultRouters is how many routers a node keeps a session with: one+ // is enough to be on the mesh, the second is there for when the first+ // restarts. Every session costs a router a connection, so a node does+ // not keep one with every router it knows of.+ DefaultRouters = 2+ // DefaultRouterShunDuration is how long a node stays off a router that+ // sent it away or did not come back: past a rollout of the router, short+ // enough that one back for good is used again.+ DefaultRouterShunDuration = 5 * time.Minute// DefaultSocketName is the local API socket the node creates in its data directory.DefaultSocketName = "agentmesh.sock")@@ -101,6 +110,23 @@ type Options struct {// an enrollment, dial the routers before giving up, RouterRedialDelay// doubling between attempts. Zero uses the default.RouterAuthAttempts int+ // Routers is how many routers the node keeps a session with, out of+ // those the control plane lists; every router when there are fewer.+ // Zero uses DefaultRouters. A session that drops is redialled, and a+ // router that does not come back is replaced from the list.+ Routers int+ // RouterSelector names labels a router must attest for the node to+ // attach to it, every pair; RouterPrefer names labels whose routers the+ // node attaches to first, the more pairs attested the earlier. Among+ // routers equal on preference the node takes the ones with the most+ // room. Both are matched against the labels /info lists for a router,+ // which are the ones the control plane attested at its enrollment.+ RouterSelector map[string]string+ RouterPrefer map[string]string+ // RouterShunDuration is how long the node stays off a router that told+ // it to go away or that did not come back on redial. Zero uses the+ // default.+ RouterShunDuration time.Duration// BiscuitTimeout bounds Datalog evaluation when verifying biscuit tokens.BiscuitTimeout time.Duration// DHT Options@@ -184,6 +210,12 @@ func (o *Options) Default() {if o.RouterAuthAttempts <= 0 {o.RouterAuthAttempts = DefaultRouterAuthAttempts}+ if o.Routers <= 0 {+ o.Routers = DefaultRouters+ }+ if o.RouterShunDuration <= 0 {+ o.RouterShunDuration = DefaultRouterShunDuration+ }if o.Reachability == "" {o.Reachability = ReachabilityPrivate}@@ -245,6 +277,12 @@ func (o *Options) Validate() error {default:return fmt.Errorf("reachability %q is not %q or %q", o.Reachability, ReachabilityPrivate, ReachabilityAuto)}+ if err := api.ValidateLabels(o.RouterSelector); err != nil {+ return fmt.Errorf("RouterSelector: %w", err)+ }+ if err := api.ValidateLabels(o.RouterPrefer); err != nil {+ return fmt.Errorf("RouterPrefer: %w", err)+ }// ed25519 verification panics on a wrong-size key, and this one comes// from a flag or FFI config.if len(o.ControlPlanePubKey) > 0 && len(o.ControlPlanePubKey) != ed25519.PublicKeySize {diff --git a/internal/node/routers.go b/internal/node/routers.gonew file mode 100644index 00000000..e80015e8--- /dev/null+++ b/internal/node/routers.go@@ -0,0 +1,349 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "context"+ "errors"+ "fmt"+ "math/rand/v2"+ "sort"+ "strings"+ "time"++ "github.com/google/agentmesh/api"+ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-libp2p/core/peerstore"+ "github.com/multiformats/go-multiaddr"+)++// routerCandidate is one router as the node knows it: where to reach it,+// the labels the control plane attested for it and its load as of its last+// lease. From /info when the control plane answered, from stored addresses+// alone when it did not.+type routerCandidate struct {+ peer.AddrInfo+ Labels map[string]string+ Connections int+ ConnectionLimit int+}++// routerTag protects an attached router's connection from the node's own+// connection manager, which otherwise trims by age and tag value and would+// take the router before a stranger.+const routerTag = "agentmesh-router"++// candidatesFromInfo reads the routers /info lists. A control plane that+// predates RouterInfo lists addresses only; those become candidates without+// labels or load.+func candidatesFromInfo(ctx context.Context, info *api.ControlPlaneInfoResponse) []routerCandidate {+ if len(info.GetRouters()) == 0 {+ return candidatesFromAddrs(ctx, parseMultiaddrs(info.GetRouterAddresses()))+ }+ var out []routerCandidate+ for _, r := range info.GetRouters() {+ for _, pi := range routerPeers(ctx, parseMultiaddrs(r.GetAddresses())) {+ out = append(out, routerCandidate{+ AddrInfo: pi,+ Labels: r.GetLabels(),+ Connections: int(r.GetConnections()),+ ConnectionLimit: int(r.GetConnectionLimit()),+ })+ }+ }+ return out+}++func candidatesFromAddrs(ctx context.Context, addrs []multiaddr.Multiaddr) []routerCandidate {+ peers := routerPeers(ctx, addrs)+ out := make([]routerCandidate, 0, len(peers))+ for _, pi := range peers {+ out = append(out, routerCandidate{AddrInfo: pi})+ }+ return out+}++// selectRouters chooses up to k routers from cands: those that attest every+// pair of selector and are not excluded, ordered by how many pairs of prefer+// they attest, then by how full they are. A little noise on the load keeps a+// fleet that reads the same /info from piling onto one router before its+// next lease reports the change. The second result is how many candidates+// passed the selector, so a caller can tell "none match" from "all taken".+func selectRouters(cands []routerCandidate, k int, selector, prefer map[string]string, exclude map[peer.ID]bool) ([]peer.AddrInfo, int) {+ type scored struct {+ routerCandidate+ preferred int+ load float64+ }+ var eligible []scored+ matched := 0+ for _, c := range cands {+ if len(selector) > 0 && !api.LabelsSatisfy(selector, c.Labels) {+ continue+ }+ matched+++ if exclude[c.ID] {+ continue+ }+ preferred := 0+ for key, value := range prefer {+ if c.Labels[key] == value {+ preferred+++ }+ }+ load := 0.5+ if c.ConnectionLimit > 0 {+ load = float64(c.Connections) / float64(c.ConnectionLimit)+ }+ load += rand.Float64() * 0.05+ eligible = append(eligible, scored{c, preferred, load})+ }+ sort.SliceStable(eligible, func(i, j int) bool {+ if eligible[i].preferred != eligible[j].preferred {+ return eligible[i].preferred > eligible[j].preferred+ }+ return eligible[i].load < eligible[j].load+ })+ if k > len(eligible) {+ k = len(eligible)+ }+ out := make([]peer.AddrInfo, 0, k)+ for _, s := range eligible[:k] {+ out = append(out, s.AddrInfo)+ }+ return out, matched+}++// setRouterCatalog replaces what the node knows about the routers and keeps+// their addresses in the peerstore, so a circuit through any of them can be+// built later.+func (n *AgentMeshNode) setRouterCatalog(cands []routerCandidate) {+ n.mu.Lock()+ n.routerCatalog = cands+ n.mu.Unlock()+ if n.Host != nil {+ for _, c := range cands {+ n.Host.Peerstore().AddAddrs(c.ID, c.Addrs, peerstore.PermanentAddrTTL)+ }+ }+}++func (n *AgentMeshNode) routerCandidates() []routerCandidate {+ n.mu.Lock()+ defer n.mu.Unlock()+ return append([]routerCandidate(nil), n.routerCatalog...)+}++// refreshRouterCatalog re-reads /info. Best effort: a node started from its+// stored addresses with no control plane in reach keeps what it has.+func (n *AgentMeshNode) refreshRouterCatalog(ctx context.Context) error {+ controlPlaneURL, err := n.Store.LoadControlPlaneURL()+ if err != nil || controlPlaneURL == "" {+ return errors.New("no control plane URL in store")+ }+ info, err := FetchControlPlaneInfo(ctx, controlPlaneURL)+ if err != nil {+ return err+ }+ if len(info.GetRouterAddresses()) == 0 && len(info.GetRouters()) == 0 {+ return errors.New("control plane lists no routers")+ }+ n.setRouterCatalog(candidatesFromInfo(ctx, info))+ return nil+}++// wantRouters is how many routers the node holds: its --routers, capped by+// the routers that attest its selector, so a selector that one router+// matches is a node that holds one router and is not short.+func (n *AgentMeshNode) wantRouters() int {+ n.mu.Lock()+ defer n.mu.Unlock()+ matched := 0+ for _, c := range n.routerCatalog {+ if len(n.config.RouterSelector) == 0 || api.LabelsSatisfy(n.config.RouterSelector, c.Labels) {+ matched+++ }+ }+ return min(n.config.Routers, matched)+}++// attachedRouters lists the routers the node keeps and holds a live+// authenticated session with.+func (n *AgentMeshNode) attachedRouters() []peer.ID {+ n.mu.Lock()+ defer n.mu.Unlock()+ var out []peer.ID+ for pid := range n.attached {+ if n.authenticatedRouters[pid] && n.Host.Network().Connectedness(pid) == network.Connected {+ out = append(out, pid)+ }+ }+ return out+}++// AttachedRouters is how many routers the node keeps a live session with.+func (n *AgentMeshNode) AttachedRouters() int { return len(n.attachedRouters()) }++// WantRouters is how many routers the node keeps.+func (n *AgentMeshNode) WantRouters() int { return n.wantRouters() }++// RefreshRouters re-reads the routers from the control plane.+func (n *AgentMeshNode) RefreshRouters(ctx context.Context) error { return n.refreshRouterCatalog(ctx) }++// attach marks a router as one the node keeps: protected from the node's+// connection manager, redialled when it drops, and offered to AutoRelay for+// a reservation.+func (n *AgentMeshNode) attach(router peer.ID) {+ n.mu.Lock()+ if n.attached == nil {+ n.attached = map[peer.ID]bool{}+ }+ n.attached[router] = true+ n.mu.Unlock()+ if n.connMgr != nil {+ n.connMgr.Protect(router, routerTag)+ }+ n.refreshRelays()+}++// detach gives a router up: the node stops redialling it and lets the+// monitor top up from the others.+func (n *AgentMeshNode) detach(router peer.ID) {+ n.mu.Lock()+ delete(n.attached, router)+ n.mu.Unlock()+ if n.connMgr != nil {+ n.connMgr.Unprotect(router, routerTag)+ }+ n.refreshRelays()+}++func (n *AgentMeshNode) isAttached(router peer.ID) bool {+ n.mu.Lock()+ defer n.mu.Unlock()+ return n.attached[router]+}++// refreshRelays offers AutoRelay the routers the node is attached to: a+// reservation needs an authenticated session, which only those have.+func (n *AgentMeshNode) refreshRelays() {+ n.mu.Lock()+ defer n.mu.Unlock()+ var relays []peer.AddrInfo+ for _, c := range n.routerCatalog {+ if n.attached[c.ID] {+ relays = append(relays, c.AddrInfo)+ }+ }+ n.currentRelays = relays+}++// pickRouters chooses k routers to attach to, from the catalog, leaving out+// those already attached and those that sent the node away. When the+// catalog has routers and none passes the selector, the warning names both+// sides, since that is a configuration to fix rather than a router to wait+// for.+func (n *AgentMeshNode) pickRouters(k int) []peer.AddrInfo {+ cands := n.routerCandidates()+ n.mu.Lock()+ exclude := make(map[peer.ID]bool, len(n.attached)+len(n.shunned))+ for pid := range n.attached {+ exclude[pid] = true+ }+ now := time.Now()+ for pid, until := range n.shunned {+ if now.Before(until) {+ exclude[pid] = true+ } else {+ delete(n.shunned, pid)+ }+ }+ n.mu.Unlock()+ picked, matched := selectRouters(cands, k, n.config.RouterSelector, n.config.RouterPrefer, exclude)+ if matched == 0 && len(cands) > 0 && len(n.config.RouterSelector) > 0 {+ var seen []string+ for _, c := range cands {+ seen = append(seen, fmt.Sprintf("%s %v", c.ID.ShortString(), c.Labels))+ }+ logger.Warnf("[AuthN] No router attests the selector %v; routers listed: %s", n.config.RouterSelector, strings.Join(seen, ", "))+ }+ return picked+}++// shun keeps the node off a router for a while: one that told it to go+// away, or one that would not come back on redial.+func (n *AgentMeshNode) shun(router peer.ID, d time.Duration) {+ n.mu.Lock()+ if n.shunned == nil {+ n.shunned = map[peer.ID]time.Time{}+ }+ n.shunned[router] = time.Now().Add(d)+ n.mu.Unlock()+}++// dialAndAttach runs authRouter against each router at once and attaches+// the ones that admit the node, delivering one outcome per router on the+// returned channel in completion order. A caller may stop reading early:+// the dials still in flight attach on their own as they land.+func (n *AgentMeshNode) dialAndAttach(ctx context.Context, routers []peer.AddrInfo, biscuitBytes []byte) <-chan error {+ results := make(chan error, len(routers))+ for _, router := range routers {+ go func() {+ err := n.authRouter(ctx, router, biscuitBytes)+ if err != nil {+ logger.Warnf("[AuthN] Failed to bootstrap and auth with router %s: %v", router.ID, err)+ } else {+ n.attach(router.ID)+ }+ results <- err+ }()+ }+ return results+}++// topUpRouters attaches to routers until the node keeps as many as it+// wants, choosing from the catalog, and returns how many it then keeps.+// Every dial is waited for here: this is the monitor's path, where there+// is no one to be ready for.+func (n *AgentMeshNode) topUpRouters(ctx context.Context) int {+ have := n.AttachedRouters()+ want := n.wantRouters()+ if have >= want {+ return have+ }+ biscuitBytes, err := n.loadIdentityForAuth()+ if err != nil {+ logger.Errorf("[AuthN] Cannot attach to routers without an identity: %v", err)+ return have+ }+ targets := n.pickRouters(want - have)+ if len(targets) == 0 {+ return have+ }+ results := n.dialAndAttach(ctx, targets, biscuitBytes)+ for range targets {+ <-results+ }+ after := n.AttachedRouters()+ if after > have {+ n.triggerReprovide()+ }+ return after+}++// TopUp attaches to more routers from what the node knows and returns how+// many it then keeps.+func (n *AgentMeshNode) TopUp(ctx context.Context) int { return n.topUpRouters(ctx) }diff --git a/internal/node/routers_test.go b/internal/node/routers_test.gonew file mode 100644index 00000000..38a8e5ab--- /dev/null+++ b/internal/node/routers_test.go@@ -0,0 +1,166 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "context"+ "testing"++ "github.com/google/agentmesh/api"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/multiformats/go-multiaddr"+)++func candidate(t *testing.T, name string, labels map[string]string, connections, limit int) routerCandidate {+ t.Helper()+ // A peer ID is derived from the name so the test can tell them apart.+ id, err := peer.Decode("12D3KooW" + name)+ if err != nil {+ // Not a real ID: build one from the name's bytes instead.+ id = peer.ID("router-" + name)+ }+ addr := multiaddr.StringCast("/ip4/127.0.0.1/tcp/4501")+ return routerCandidate{+ AddrInfo: peer.AddrInfo{ID: id, Addrs: []multiaddr.Multiaddr{addr}},+ Labels: labels,+ Connections: connections,+ ConnectionLimit: limit,+ }+}++func ids(infos []peer.AddrInfo) []peer.ID {+ out := make([]peer.ID, 0, len(infos))+ for _, pi := range infos {+ out = append(out, pi.ID)+ }+ return out+}++// TestSelectRouters pins the choice of routers: the selector is a+// requirement on every pair, the preference orders by pairs attested, load+// orders among equals, and routers already attached or shunned are left+// out. The second result tells "none match" from "all taken".+func TestSelectRouters(t *testing.T) {+ euEmpty := candidate(t, "eu-empty", map[string]string{"region": "eu", "zone": "eu-a"}, 100, 4000)+ euFull := candidate(t, "eu-full", map[string]string{"region": "eu", "zone": "eu-b"}, 3900, 4000)+ usEmpty := candidate(t, "us-empty", map[string]string{"region": "us"}, 0, 4000)+ unlabelled := candidate(t, "plain", nil, 0, 0)+ all := []routerCandidate{euFull, usEmpty, unlabelled, euEmpty}++ t.Run("by load when nothing else is said", func(t *testing.T) {+ picked, matched := selectRouters(all, 2, nil, nil, nil)+ if matched != 4 {+ t.Fatalf("matched = %d, want all 4", matched)+ }+ got := ids(picked)+ // The full router comes last; the unlabelled one with unknown load+ // sits in the middle at 0.5.+ if len(got) != 2 || got[0] == euFull.ID || got[1] == euFull.ID {+ t.Fatalf("picked %v, want the two with the most room", got)+ }+ })++ t.Run("selector is a requirement on every pair", func(t *testing.T) {+ picked, matched := selectRouters(all, 3, map[string]string{"region": "eu", "zone": "eu-a"}, nil, nil)+ if matched != 1 || len(picked) != 1 || picked[0].ID != euEmpty.ID {+ t.Fatalf("picked %v (matched %d), want only eu-a", ids(picked), matched)+ }+ _, matched = selectRouters(all, 3, map[string]string{"region": "apac"}, nil, nil)+ if matched != 0 {+ t.Fatalf("matched = %d for a region no router attests, want 0", matched)+ }+ })++ t.Run("preference orders before load", func(t *testing.T) {+ picked, _ := selectRouters(all, 2, nil, map[string]string{"region": "eu"}, nil)+ got := ids(picked)+ if len(got) != 2 || got[0] != euEmpty.ID || got[1] != euFull.ID {+ t.Fatalf("picked %v, want the eu routers first, the emptier one before the full one", got)+ }+ // More pairs attested ranks higher.+ picked, _ = selectRouters(all, 1, nil, map[string]string{"region": "eu", "zone": "eu-b"}, nil)+ if len(picked) != 1 || picked[0].ID != euFull.ID {+ t.Fatalf("picked %v, want eu-b for attesting both preferred pairs although full", ids(picked))+ }+ })++ t.Run("attached and shunned routers are left out, but still count as matching", func(t *testing.T) {+ exclude := map[peer.ID]bool{euEmpty.ID: true, usEmpty.ID: true}+ picked, matched := selectRouters(all, 5, nil, nil, exclude)+ if matched != 4 || len(picked) != 2 {+ t.Fatalf("picked %d (matched %d), want 2 of 4", len(picked), matched)+ }+ for _, pi := range picked {+ if exclude[pi.ID] {+ t.Fatalf("picked an excluded router %s", pi.ID)+ }+ }+ })++ t.Run("k is clamped", func(t *testing.T) {+ picked, _ := selectRouters(all[:1], 3, nil, nil, nil)+ if len(picked) != 1 {+ t.Fatalf("picked %d from 1 candidate, want 1", len(picked))+ }+ })+}++// A control plane that predates RouterInfo lists addresses alone; they+// become candidates with no labels and unknown load, so a selector matches+// none and a plain node still has something to attach to.+func TestCandidatesFromInfoFallsBackToAddresses(t *testing.T) {+ id, _ := peer.Decode("12D3KooWGvdRCJLYATauVWfsieF2j3a2wXZoEQJUS2MsvRdDtgLM")+ addr := "/ip4/127.0.0.1/tcp/4501/p2p/" + id.String()+ old := &api.ControlPlaneInfoResponse{RouterAddresses: []string{addr}}+ cands := candidatesFromInfo(context.Background(), old)+ if len(cands) != 1 || cands[0].ID != id || cands[0].Labels != nil || cands[0].ConnectionLimit != 0 {+ t.Fatalf("candidates from an address-only /info = %+v, want one unlabelled router", cands)+ }++ current := &api.ControlPlaneInfoResponse{+ RouterAddresses: []string{addr},+ Routers: []*api.RouterInfo{{+ PeerId: id.String(), Addresses: []string{addr},+ Labels: map[string]string{"region": "eu"}, Connections: 10, ConnectionLimit: 4000,+ }},+ }+ cands = candidatesFromInfo(context.Background(), current)+ if len(cands) != 1 || cands[0].Labels["region"] != "eu" || cands[0].Connections != 10 {+ t.Fatalf("candidates from /info = %+v, want the labelled router with its load", cands)+ }+}++// A node wants as many routers as it is configured for, but no more than+// attest its selector: with one matching router it is complete at one, and+// the monitor does not keep topping up.+func TestWantRoutersCountsSelectorMatches(t *testing.T) {+ n := &AgentMeshNode{config: Options{Routers: 2}}+ n.routerCatalog = []routerCandidate{+ candidate(t, "a", map[string]string{"site": "a"}, 0, 100),+ candidate(t, "b", map[string]string{"site": "b"}, 0, 100),+ candidate(t, "c", nil, 0, 100),+ }+ if got := n.wantRouters(); got != 2 {+ t.Fatalf("wantRouters without selector = %d, want 2", got)+ }+ n.config.RouterSelector = map[string]string{"site": "a"}+ if got := n.wantRouters(); got != 1 {+ t.Fatalf("wantRouters with selector site=a = %d, want 1", got)+ }+ n.config.RouterSelector = map[string]string{"site": "z"}+ if got := n.wantRouters(); got != 0 {+ t.Fatalf("wantRouters with a selector no router attests = %d, want 0", got)+ }+}diff --git a/site/content/docs/reference/agentmesh-node.md b/site/content/docs/reference/agentmesh-node.mdindex 619104cb..2347d777 100644--- a/site/content/docs/reference/agentmesh-node.md+++ b/site/content/docs/reference/agentmesh-node.md@@ -110,6 +110,9 @@ explains how to enroll.| `--listen` | `/ip4/0.0.0.0/udp/5001/quic-v1`, `/ip4/0.0.0.0/tcp/5002` | libp2p listen addresses. Repeatable. || `--allow-loopback`, `--announce-private`, `--router-connect-timeout` | | As for `join`. || `--reachability` | `private` | How the node decides whether peers can dial it directly. `private` always holds a relay reservation with the routers and is reached through them, which works behind any NAT. `auto` lets AutoNAT decide from dial-backs by the routers: a node they reach on a public address advertises it and holds no reservation, so direct dials replace relayed ones. Use `auto` only where the routers are on the public side of every NAT the node is behind; a router in the same cluster or LAN can reach a node the rest of the mesh cannot, and would make it unreachable. |+| `--routers` | `2` | Routers to keep a session with, out of those the control plane lists; every router when there are fewer. One is enough to be on the mesh, the second is there for when the first restarts. A session that drops is redialled at 2 s, 4 s and so on; a router that does not come back is replaced from the list. A call reaches a peer on any router, attached or not. |+| `--router-selector` | none | Attach only to routers whose credential attests this `key=value` label, repeatable, every pair required: `--router-selector region=eu`. The labels are the ones the control plane lists on `/info`, attested at the router's enrollment. A selector no router attests leaves the node retrying as if no router answered, with a warning naming the selector and the labels it saw. |+| `--router-prefer` | none | Attach to routers attesting this `key=value` label before others, repeatable; a router attesting more of the pairs comes earlier, and among equals the one with the most room. `--router-prefer zone=eu-a` keeps a node on its zone's routers while they have capacity and lets it fall to the others. || `--enable-relay` | `false` | Relay traffic for other peers. || `--mesh` | `public-mesh` | Mesh federation name. Must match the routers' name. || `--discovery-interval` | `30s` | DHT discovery polling interval. |