Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: authenticate a listed peer router whenever it is not admitted
Antonio Ojeae3e453fffe54
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex dae889e7..4322ae4f 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -1425,6 +1425,9 @@ func (s *Server) HandleRouterLease(w http.ResponseWriter, r *http.Request) {http.Error(w, "Invalid load: connections and connection_limit must not be negative", http.StatusBadRequest)return}+ if len(req.Addresses) == 0 {+ logger.Warnw("Router leased without addresses: it is listed but members cannot reach it", "peer_id", canonical)+ }// The lease lasts the control plane's duration unless the router asked// for less: a router that is stopping says how long it expects to bediff --git a/internal/identity/biscuit.go b/internal/identity/biscuit.goindex 00c93091..b28a9059 100644--- a/internal/identity/biscuit.go+++ b/internal/identity/biscuit.go@@ -551,6 +551,13 @@ func VerifyBiscuitAndGetExpiry(biscuitData []byte, expectedPeer peer.ID, trustedreturn expiry, err}+// VerifyBiscuitAdmission is VerifyBiscuit that reports both the key that+// verified the token and when it lapses, for a caller that caches the+// admission and goes on to evaluate the token under that key.+func VerifyBiscuitAdmission(biscuitData []byte, expectedPeer peer.ID, trustedPublicKeys []ed25519.PublicKey, timeout time.Duration) (*biscuit.Biscuit, ed25519.PublicKey, time.Time, error) {+ return verifyBiscuit(biscuitData, expectedPeer, trustedPublicKeys, timeout)+}+func verifyBiscuit(biscuitData []byte, expectedPeer peer.ID, trustedPublicKeys []ed25519.PublicKey, timeout time.Duration) (*biscuit.Biscuit, ed25519.PublicKey, time.Time, error) {b, rules, err := UnmarshalInbound(biscuitData)if err != nil {diff --git a/internal/router/debug.go b/internal/router/debug.gonew file mode 100644index 00000000..272470fd--- /dev/null+++ b/internal/router/debug.go@@ -0,0 +1,69 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package router++import (+ "encoding/json"+ "net/http"++ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+)++// peerView is what the router knows about one peer: the addresses it would+// hand out for it in a DHT answer, and the standing the peer has here.+// Unversioned diagnostics on the operator listener, parsed by no client.+type peerView struct {+ PeerID string `json:"peer_id"`+ Connectedness string `json:"connectedness"`+ Authenticated bool `json:"authenticated"`+ PeerRouter bool `json:"peer_router"`+ Connections []string `json:"connections"`+ Addrs []string `json:"addrs"`+}++// handleDebugPeer serves GET /debug/peer?peer_id=<id>.+func (r *Router) handleDebugPeer(w http.ResponseWriter, req *http.Request) {+ if !r.isReady.Load() {+ http.Error(w, "router is not online yet", http.StatusServiceUnavailable)+ return+ }+ p, err := peer.Decode(req.URL.Query().Get("peer_id"))+ if err != nil {+ http.Error(w, "peer_id: "+err.Error(), http.StatusBadRequest)+ return+ }+ _, isRouter := r.peerRouters.Load(p)+ view := peerView{+ PeerID: p.String(),+ Connectedness: r.Host.Network().Connectedness(p).String(),+ Authenticated: r.isPeerAuthenticated(p),+ PeerRouter: isRouter,+ Connections: []string{},+ Addrs: []string{},+ }+ for _, c := range r.Host.Network().ConnsToPeer(p) {+ dir := "inbound"+ if c.Stat().Direction == network.DirOutbound {+ dir = "outbound"+ }+ view.Connections = append(view.Connections, dir+" "+c.RemoteMultiaddr().String())+ }+ for _, a := range r.Host.Peerstore().Addrs(p) {+ view.Addrs = append(view.Addrs, a.String())+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(view)+}diff --git a/internal/router/metrics.go b/internal/router/metrics.goindex 0f1e59a9..42bc1f7f 100644--- a/internal/router/metrics.go+++ b/internal/router/metrics.go@@ -113,6 +113,7 @@ type routerStateCollector struct {readyDesc *prometheus.DescdrainingDesc *prometheus.DescauthenticatedDesc *prometheus.Desc+ peerRoutersDesc *prometheus.DescconnectedDesc *prometheus.DescconnectionsDesc *prometheus.DescwatermarkDesc *prometheus.Desc@@ -141,6 +142,10 @@ func newRouterStateCollector(r *Router) *routerStateCollector {"agentmesh_router_authenticated_peers","Connected peers that have completed the mesh authentication handshake",nil, nil),+ peerRoutersDesc: prometheus.NewDesc(+ "agentmesh_router_peer_routers",+ "Peer routers known, from the control plane's list or a handshake with the router role: all of them, and those currently authenticated",+ []string{"state"}, nil),connectedDesc: prometheus.NewDesc("agentmesh_router_connected_peers","Peers with an open libp2p connection, authenticated or not",@@ -174,7 +179,7 @@ func newRouterStateCollector(r *Router) *routerStateCollector {func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {for _, d := range []*prometheus.Desc{- c.infoDesc, c.readyDesc, c.drainingDesc, c.authenticatedDesc, c.connectedDesc, c.connectionsDesc, c.watermarkDesc, c.relayLimitDesc,+ c.infoDesc, c.readyDesc, c.drainingDesc, c.authenticatedDesc, c.peerRoutersDesc, c.connectedDesc, c.connectionsDesc, c.watermarkDesc, c.relayLimitDesc,c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc, connsPerSourceIPDesc,} {ch <- d@@ -203,6 +208,9 @@ func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 1)ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, r.Host.ID().String())ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.authenticatedPeers)))+ listed, federated := r.peerRouterCounts()+ ch <- prometheus.MustNewConstMetric(c.peerRoutersDesc, prometheus.GaugeValue, float64(listed), "listed")+ ch <- prometheus.MustNewConstMetric(c.peerRoutersDesc, prometheus.GaugeValue, float64(federated), "authenticated")ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(r.Host.Network().Peers())))var inbound, outbound intfor _, conn := range r.Host.Network().Conns() {@@ -234,7 +242,8 @@ func syncMapLen(m *sync.Map) int {return n}-// serveMetrics opens the operator listener: /metrics, /healthz and /readyz.+// serveMetrics opens the operator listener: /metrics, /healthz, /readyz and+// /debug/peer, the router's view of one peer.// It is off unless an address is configured, and never shares a port with the// libp2p listeners, so it can stay cluster-internal while the mesh port is// public. libp2p's own metrics land in the default registry, so exposing it@@ -263,6 +272,7 @@ func (r *Router) serveMetrics(addr string) error {}w.WriteHeader(http.StatusOK)})+ mux.HandleFunc("GET /debug/peer", r.handleDebugPeer)r.metricsServer = &http.Server{Handler: mux,diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.goindex 6b5ae0d0..2f98750a 100644--- a/internal/router/metrics_test.go+++ b/internal/router/metrics_test.go@@ -16,6 +16,7 @@ package routerimport ("context"+ "encoding/json""io""net/http""strconv"@@ -86,6 +87,9 @@ func TestRouterMetricsListener(t *testing.T) {r.authenticatedPeers.Store(peer.ID("peer-a"), true)r.authenticatedPeers.Store(peer.ID("peer-b"), true)r.bannedPeers.Store(peer.ID("peer-x"), true)+ // Two peer routers listed, of which one is authenticated.+ r.peerRouters.Store(peer.ID("peer-a"), true)+ r.peerRouters.Store(peer.ID("peer-r"), true)expiry := time.Now().Add(time.Hour).Truncate(time.Second)r.keysMu.Lock()r.credential.expiration = expiry@@ -110,6 +114,8 @@ func TestRouterMetricsListener(t *testing.T) {`agentmesh_router_relay_limit{limit="circuits_per_peer"} 1024`,"agentmesh_router_inbound_connections_refused_total","agentmesh_router_authenticated_peers 2",+ `agentmesh_router_peer_routers{state="listed"} 2`,+ `agentmesh_router_peer_routers{state="authenticated"} 1`,"agentmesh_router_banned_peers 1","agentmesh_router_connected_peers 0","agentmesh_router_dht_routing_table_size 0",@@ -126,6 +132,32 @@ func TestRouterMetricsListener(t *testing.T) {}}+ // /debug/peer is the router's view of one peer: its standing here and+ // the addresses the router would hand out for it.+ other, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = other.Close() }()+ if err := other.Connect(ctx, peer.AddrInfo{ID: host.ID(), Addrs: host.Addrs()}); err != nil {+ t.Fatal(err)+ }+ r.authenticatedPeers.Store(other.ID(), time.Now().Add(time.Hour))+ code, body = getBody(t, base+"/debug/peer?peer_id="+other.ID().String())+ if code != http.StatusOK {+ t.Fatalf("/debug/peer = %d: %s", code, body)+ }+ var view peerView+ if err := json.Unmarshal([]byte(body), &view); err != nil {+ t.Fatalf("/debug/peer: %v: %s", err, body)+ }+ if view.Connectedness != "Connected" || !view.Authenticated || view.PeerRouter || len(view.Connections) != 1 || len(view.Addrs) == 0 {+ t.Errorf("/debug/peer for a connected, authenticated member: %+v", view)+ }+ if code, _ := getBody(t, base+"/debug/peer?peer_id=nonsense"); code != http.StatusBadRequest {+ t.Errorf("/debug/peer with a bad peer id = %d, want 400", code)+ }+if err := r.Close(); err != nil {t.Fatal(err)}diff --git a/internal/router/router.go b/internal/router/router.goindex 794fa17f..9c83059f 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -311,7 +311,9 @@ type Router struct {// identityLoggedAt is when the "Router Online" line was last written.identityLoggedAt atomic.Pointer[time.Time]- authFailureLog perPeerLogLimiter+ // leasedAddrs is the address list of the last accepted lease, joined.+ leasedAddrs atomic.Pointer[string]+ authFailureLog perPeerLogLimiter}// NewRouter initializes the router.@@ -1151,12 +1153,29 @@ func (r *Router) sendLease(ctx context.Context, ttl *time.Duration, attempts int} else {logger.Debugf("Lease renewed successfully. Expires at: %s", leaseResp.GetExpireTime().AsTime().Format(time.RFC3339))leaseRenewalsTotal.WithLabelValues(leaseOK).Inc()+ r.logLeasedAddrs(addrs)r.logIdentityPeriodically()}return}}+// logLeasedAddrs writes the addresses the control plane now lists for this+// router each time they differ from the last lease: members and peer routers+// reach the router only through these, and a lease without any leaves it+// listed but unreachable.+func (r *Router) logLeasedAddrs(addrs []string) {+ joined := strings.Join(addrs, ",")+ if last := r.leasedAddrs.Swap(&joined); last != nil && *last == joined {+ return+ }+ if len(addrs) == 0 {+ logger.Warn("Lease advertises no addresses: the control plane lists this router but nothing can reach it")+ return+ }+ logger.Infof("Lease advertises %d address(es): %v", len(addrs), addrs)+}+// identityLogInterval is how often the router restates who it is. Container// logs rotate, and the startup line is the first to go; an operator reading// the log should still find the peer ID within the last hour of it.@@ -1277,37 +1296,66 @@ func (r *Router) connectBootstrapRouters() {for _, resolved := range resolvedAddrs {pi, err := peer.AddrInfoFromP2pAddr(resolved)- if err != nil || pi.ID == r.Host.ID() {+ if err != nil {+ logger.Warnf("[Federation] Ignoring router address %s: %v", resolved, err)+ continue+ }+ if pi.ID == r.Host.ID() {continue}r.peerRouters.Store(pi.ID, true)r.Host.ConnManager().Protect(pi.ID, federationTag)-- if len(r.Host.Network().ConnsToPeer(pi.ID)) == 0 {- logger.Infof("[Federation] Connecting to peer router: %s via %s", pi.ID, resolved)- // Create a timeout context- connectCtx, cancel := context.WithTimeout(r.ctx, 10*time.Second)- if err := r.Host.Connect(connectCtx, *pi); err == nil {- // Initiate stream to perform mutual auth handshake- s, err := r.Host.NewStream(connectCtx, pi.ID, api.AuthProtocolID)- if err == nil {- if err := r.performMutualAuth(s); err != nil {- logger.Errorf("[Federation] Mutual auth handshake failed with %s: %v", pi.ID, err)- _ = s.Reset()- } else {- logger.Infof("[Federation] Mutually authenticated with peer router %s", pi.ID)- _ = s.Close()- }- } else {- logger.Errorf("[Federation] Failed to open auth stream to %s: %v", pi.ID, err)- }- } else {- logger.Errorf("[Federation] Failed to connect to %s: %v", pi.ID, err)- }- cancel()+ if r.isPeerAuthenticated(pi.ID) {+ continue}+ r.authenticatePeerRouter(*pi)+ }+ }+}++// authenticatePeerRouter runs the mutual handshake with a router the control+// plane lists, over the connection to it if one is open (the DHT commonly+// opens one first), dialling otherwise. Authentication is what decides, not+// the connection: a connection the DHT made is not an authenticated one,+// and an admission that expired is renewed on the next pass.+func (r *Router) authenticatePeerRouter(pi peer.AddrInfo) {+ ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second)+ defer cancel()+ if len(r.Host.Network().ConnsToPeer(pi.ID)) == 0 {+ logger.Infof("[Federation] Connecting to peer router: %s via %v", pi.ID, pi.Addrs)+ if err := r.Host.Connect(ctx, pi); err != nil {+ logger.Errorf("[Federation] Failed to connect to %s: %v", pi.ID, err)+ return}+ } else {+ logger.Infof("[Federation] Authenticating with peer router %s over its open connection", pi.ID)+ }+ s, err := r.Host.NewStream(ctx, pi.ID, api.AuthProtocolID)+ if err != nil {+ logger.Errorf("[Federation] Failed to open auth stream to %s: %v", pi.ID, err)+ return+ }+ if err := r.performMutualAuth(s); err != nil {+ logger.Errorf("[Federation] Mutual auth handshake failed with %s: %v", pi.ID, err)+ _ = s.Reset()+ return}+ logger.Infof("[Federation] Mutually authenticated with peer router %s", pi.ID)+ _ = s.Close()+}++// peerRouterCounts is how many peer routers this router knows, from the+// control plane's list or a handshake that proved the router role, and how+// many of those hold a current authentication.+func (r *Router) peerRouterCounts() (listed, authenticated int) {+ r.peerRouters.Range(func(k, _ any) bool {+ listed+++ if r.isPeerAuthenticated(k.(peer.ID)) {+ authenticated+++ }+ return true+ })+ return listed, authenticated}// recoverStreamHandler isolates a panic while processing untrusted peer bytes@@ -1376,7 +1424,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {}// Verify Biscuit and record its expiration so relayACL drops the session on time.- expiry, err := identity.VerifyBiscuitAndGetExpiry(exchange.Biscuit, remotePeer, r.getTrustedPublicKeys(), r.config.BiscuitTimeout)+ b, verifyingKey, expiry, err := identity.VerifyBiscuitAdmission(exchange.Biscuit, remotePeer, r.getTrustedPublicKeys(), r.config.BiscuitTimeout)if err != nil {// A peer stuck on a bad credential retries every few seconds for// days; one line a minute per peer keeps the log readable and the@@ -1392,6 +1440,14 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {r.admitPeer(remotePeer, expiry)authHandshakesTotal.WithLabelValues(handshakeOK).Inc()logger.Infof("[AuthN] Successfully authenticated peer %s", remotePeer)+ // A router that proves the router role is a peer router from here on,+ // not from the next federation pass: it is never a member to shed.+ if identity.RequireRole(b, verifyingKey, api.RoleRouter, r.config.BiscuitTimeout) == nil {+ if _, known := r.peerRouters.LoadOrStore(remotePeer, true); !known {+ logger.Infof("[Federation] Peer router %s authenticated with us", remotePeer)+ r.Host.ConnManager().Protect(remotePeer, federationTag)+ }+ }// Send mutual response (our biscuit)r.keysMu.RLock()diff --git a/internal/router/router_test.go b/internal/router/router_test.goindex 31a6e40c..4f7b2ccd 100644--- a/internal/router/router_test.go+++ b/internal/router/router_test.go@@ -30,6 +30,7 @@ import ("path/filepath""strings""sync"+ "sync/atomic""testing""time"@@ -392,8 +393,8 @@ func TestRouterFederation(t *testing.T) {// Wait for both routers to renew their lease so they are both in control plane DBtime.Sleep(3 * time.Second)- // Trigger connection sync manually to form the federation link- r1.connectBootstrapRouters()+ // Router 2's pass alone forms the link: Router 1 learns Router 2 is a+ // router from the handshake, not from a pass of its own.r2.connectBootstrapRouters()// Wait a moment for stream handshakes to complete@@ -413,6 +414,15 @@ func TestRouterFederation(t *testing.T) {if _, authenticated := r2.authenticatedPeers.Load(r1.Host.ID()); !authenticated {t.Errorf("Router 1 is not authenticated in Router 2's peers")}++ for name, r := range map[string]*Router{"Router 1": r1, "Router 2": r2} {+ if listed, authenticated := r.peerRouterCounts(); listed != 1 || authenticated != 1 {+ t.Errorf("%s knows %d peer router(s), %d authenticated; want 1 and 1", name, listed, authenticated)+ }+ if members := r.members(); len(members) != 0 {+ t.Errorf("%s counts %v as members; a peer router is not one", name, members)+ }+ }}func TestRouterProactiveTokenRefresh(t *testing.T) {@@ -1043,6 +1053,76 @@ func TestPerformMutualAuth(t *testing.T) {}}+// A connection to a peer router is commonly open before the federation pass+// reaches it (the DHT dials routers it learns of), and an admission expires.+// Neither leaves the pair unauthenticated: the pass authenticates whenever+// the peer is not currently admitted, over whatever connection is open.+func TestFederationAuthenticatesListedRoutersNotCurrentlyAdmitted(t *testing.T) {+ pub, priv, err := ed25519.GenerateKey(nil)+ if err != nil {+ t.Fatal(err)+ }+ peerRouter, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = peerRouter.Close() }()+ peerBiscuit, err := identity.MintBootstrapBiscuitToken(priv, peerRouter.ID(), api.RoleRouter, time.Now().Add(time.Hour), nil, nil)+ if err != nil {+ t.Fatal(err)+ }+ var handshakes atomic.Int32+ peerRouter.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {+ defer func() { _ = s.Close() }()+ reader := msgio.NewVarintReaderSize(s, 1024*64)+ msg, err := reader.ReadMsg()+ if err != nil {+ return+ }+ reader.ReleaseMsg(msg)+ handshakes.Add(1)+ respBytes, _ := proto.Marshal(&api.AuthResponse{Success: true, Biscuit: peerBiscuit})+ _ = msgio.NewVarintWriter(s).WriteMsg(respBytes)+ })++ r := newGoAwayRouter(t, 100)+ r.ctx = context.Background()+ r.credential = credential{biscuit: []byte("router-biscuit")}+ r.trustedPublicKeys = []ed25519.PublicKey{pub}+ r.config.BiscuitTimeout = time.Second+ r.config.RequiredRole = api.RoleRouter+ pi := peer.AddrInfo{ID: peerRouter.ID(), Addrs: peerRouter.Addrs()}++ // The DHT got there first.+ if err := r.Host.Connect(context.Background(), pi); err != nil {+ t.Fatal(err)+ }+ r.peerRouters.Store(pi.ID, true)+ if listed, authenticated := r.peerRouterCounts(); listed != 1 || authenticated != 0 {+ t.Fatalf("before the pass: listed %d, authenticated %d; want 1, 0", listed, authenticated)+ }+ r.authenticatePeerRouter(pi)+ if !r.isPeerAuthenticated(pi.ID) {+ t.Fatal("peer router connected by the DHT was not authenticated")+ }+ if listed, authenticated := r.peerRouterCounts(); listed != 1 || authenticated != 1 {+ t.Fatalf("after the pass: listed %d, authenticated %d; want 1, 1", listed, authenticated)+ }+ if got := handshakes.Load(); got != 1 {+ t.Fatalf("handshakes with the peer router: %d, want 1", got)+ }++ // The admission lapsed; it is renewed, not left to the connection.+ r.authenticatedPeers.Store(pi.ID, time.Now().Add(-time.Second))+ if _, authenticated := r.peerRouterCounts(); authenticated != 0 {+ t.Fatal("an expired admission counts as authenticated")+ }+ r.authenticatePeerRouter(pi)+ if !r.isPeerAuthenticated(pi.ID) || handshakes.Load() != 2 {+ t.Fatalf("expired admission not renewed: authenticated %v, handshakes %d", r.isPeerAuthenticated(pi.ID), handshakes.Load())+ }+}+// The router's /mesh/auth handler is reachable by any internet peer. An idle// stream must be closed on the router's schedule, and a peer handshaking in a// tight loop must be cut off before every frame is verified.diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex cf97c549..a7cb5fdc 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -50,7 +50,7 @@ replica count.| `--relay-max-circuits` | `1024` | Relayed connections one peer may hold at once through this router, as caller or as destination. A service that many members call at once needs this many open circuits. || `--shutdown-lease-ttl` | `30s` | Sent in the router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. The default covers a pod that restarts in place. A router that is being removed for good can send `1s`. `0` sends nothing and the lease expires on the control plane's schedule. || `--auth-deadline` | `1m` | How long an inbound connection may stay open without its peer passing the handshake. A member handshakes the moment it connects; a member attached elsewhere whose DHT client came to query is done in less, as is the control plane's event publisher, which presents a credential of its own. What is left at the deadline is closed, so the connections a router holds are its members' and the watermarks mean members. `0` keeps every connection. |-| `--metrics-addr` | off | Serve `/metrics`, `/healthz` and `/readyz` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. |+| `--metrics-addr` | off | Serve `/metrics`, `/healthz`, `/readyz` and `/debug/peer` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. || `--log-level` | `info` | `debug`, `info`, `warn`, `error`. `LOG_FORMAT=json` selects JSON output. |## What it does at run time@@ -114,6 +114,7 @@ The metrics address exposes, next to the Go runtime metrics:| `agentmesh_router_conns_per_source_ip_limit` | The per-address cap in force. || `agentmesh_router_relay_limit{limit}` | The relay budget in force: `reservations`, `reservations_per_ip` and `circuits_per_peer`. || `agentmesh_router_connected_peers`, `agentmesh_router_authenticated_peers`, `agentmesh_router_banned_peers` | Peers in each state. |+| `agentmesh_router_peer_routers{state}` | Other routers this one knows, from the control plane's list or a handshake that proved the router role: `listed`, and `authenticated` for those it currently holds a mutual authentication with. The two differ while a federation pass is pending or a peer router is down. || `agentmesh_router_dht_routing_table_size` | DHT routing table size. || `agentmesh_router_auth_handshakes_total{result}`, `agentmesh_router_lease_renewals_total{result}` | Handshakes and lease renewals by outcome. || `agentmesh_router_goaway_sent_total{reason,result}` | Members sent away, by reason (`DRAINING`, `OVERLOADED`) and outcome: `ok` when the member took the message, `unsupported` when it does not speak the protocol and was closed instead, `failed` when the stream broke. A rising `OVERLOADED` count on a router that is not draining is a fleet that has outgrown its routers. |@@ -126,3 +127,10 @@ approaches the `high` watermark or open relay reservations approach`agentmesh_router_relay_limit{limit="reservations"}` on every router that is notdraining; both are the inputs for an autoscaler. `/healthz` and `/readyz`on the same address are the probes to use in a pod spec.++`GET /debug/peer?peer_id=<id>` on the same address is the router's view of+one peer: its connectedness, whether it is authenticated here and whether it+is a peer router, its open connections, and the addresses the router holds+for it, which are the ones it hands out in a DHT answer. It is a diagnostic+for an operator reading why a member cannot reach another, and its shape is+not versioned.diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.goindex 0f345660..365dc447 100644--- a/tests/integration/router_attachment_test.go+++ b/tests/integration/router_attachment_test.go@@ -340,10 +340,24 @@ bindings:"--insecure-skip-tls-verify","--mesh-reconnect-interval", "500ms",)+ cpLog := filepath.Join(m.tmpDir, "control-plane.log")+ cpLogFile, err := os.Create(cpLog)+ if err != nil {+ t.Fatal(err)+ }+ cpCmd.Stdout, cpCmd.Stderr = cpLogFile, cpLogFileif err := cpCmd.Start(); err != nil {t.Fatalf("start control plane: %v", err)}- t.Cleanup(func() { _ = cpCmd.Process.Kill(); _ = cpCmd.Wait() })+ t.Cleanup(func() {+ _ = cpCmd.Process.Kill()+ _ = cpCmd.Wait()+ _ = cpLogFile.Close()+ if t.Failed() {+ out, _ := os.ReadFile(cpLog)+ t.Logf("--- control plane ---\n%s", out)+ }+ })waitForControlPlane(t, m.cpPort)injectPolicyYAML(t, m.cpPort, testAdminToken, policyFile)return m@@ -526,9 +540,13 @@ func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {for _, s := range sites {routers[s] = m.startRouter(t, "router-"+s, "site="+s)}- // Federation on the first pass: every router finds the ones before it.- waitForEqual(t, "routers connected to router c", func() []string { return routersConnectedTo(t, m.cpPort, routers["c"].peerID) },- peerIDStrings(routers["a"].peerID, routers["b"].peerID), 10*time.Second)+ // Federation on the first pass: every router finds the ones before it,+ // and authenticates them whether or not the DHT connected them first.+ for _, s := range sites {+ waitForEqual(t, "peer routers router "+s+" authenticated", func() []string {+ return []string{fmt.Sprint(routers[s].gauge(t, `agentmesh_router_peer_routers{state="authenticated"}`))}+ }, []string{"2"}, 10*time.Second)+ }nodes := map[string]*backgroundNode{}for _, s := range sites {@@ -574,6 +592,23 @@ func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {}}if t.Failed() {+ known := map[string]string{}+ for _, s := range sites {+ known[routers[s].peerID.String()] = "router " + s+ known[nodes[s].peerID.String()] = "node " + s+ }+ for _, lease := range fetchAdminStatus(t, m.cpPort, testAdminToken).GetActiveRouters() {+ var peers []string+ for _, p := range lease.GetConnectedPeers() {+ name := known[p]+ if name == "" {+ name = "other " + p[len(p)-6:]+ }+ peers = append(peers, name)+ }+ slices.Sort(peers)+ t.Logf("%s leases %d address(es), connected to: %v", known[lease.GetPeerId()], len(lease.GetAddresses()), peers)+ }for _, s := range sites {out := nodes[s].log()if i := strings.LastIndex(out, "[AuthN] Successfully authenticated"); i > 0 {