Skip to content

Agent Mesh

router: authenticate a listed peer router whenever it is not admitted

Antonio Ojeae3e453fffe54
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index dae889e7..4322ae4f 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -1425,6 +1425,9 @@ func (s *Server) HandleRouterLease(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Invalid load: connections and connection_limit must not be negative", http.StatusBadRequest)
return
}
+ if len(req.Addresses) == 0 {
+ logger.Warnw("Router leased without addresses: it is listed but members cannot reach it", "peer_id", canonical)
+ }
// The lease lasts the control plane's duration unless the router asked
// for less: a router that is stopping says how long it expects to be
diff --git a/internal/identity/biscuit.go b/internal/identity/biscuit.go
index 00c93091..b28a9059 100644
--- a/internal/identity/biscuit.go
+++ b/internal/identity/biscuit.go
@@ -551,6 +551,13 @@ func VerifyBiscuitAndGetExpiry(biscuitData []byte, expectedPeer peer.ID, trusted
return expiry, err
}
+// VerifyBiscuitAdmission is VerifyBiscuit that reports both the key that
+// verified the token and when it lapses, for a caller that caches the
+// admission and goes on to evaluate the token under that key.
+func VerifyBiscuitAdmission(biscuitData []byte, expectedPeer peer.ID, trustedPublicKeys []ed25519.PublicKey, timeout time.Duration) (*biscuit.Biscuit, ed25519.PublicKey, time.Time, error) {
+ return verifyBiscuit(biscuitData, expectedPeer, trustedPublicKeys, timeout)
+}
+
func verifyBiscuit(biscuitData []byte, expectedPeer peer.ID, trustedPublicKeys []ed25519.PublicKey, timeout time.Duration) (*biscuit.Biscuit, ed25519.PublicKey, time.Time, error) {
b, rules, err := UnmarshalInbound(biscuitData)
if err != nil {
diff --git a/internal/router/debug.go b/internal/router/debug.go
new file mode 100644
index 00000000..272470fd
--- /dev/null
+++ b/internal/router/debug.go
@@ -0,0 +1,69 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "encoding/json"
+ "net/http"
+
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+// peerView is what the router knows about one peer: the addresses it would
+// hand out for it in a DHT answer, and the standing the peer has here.
+// Unversioned diagnostics on the operator listener, parsed by no client.
+type peerView struct {
+ PeerID string `json:"peer_id"`
+ Connectedness string `json:"connectedness"`
+ Authenticated bool `json:"authenticated"`
+ PeerRouter bool `json:"peer_router"`
+ Connections []string `json:"connections"`
+ Addrs []string `json:"addrs"`
+}
+
+// handleDebugPeer serves GET /debug/peer?peer_id=<id>.
+func (r *Router) handleDebugPeer(w http.ResponseWriter, req *http.Request) {
+ if !r.isReady.Load() {
+ http.Error(w, "router is not online yet", http.StatusServiceUnavailable)
+ return
+ }
+ p, err := peer.Decode(req.URL.Query().Get("peer_id"))
+ if err != nil {
+ http.Error(w, "peer_id: "+err.Error(), http.StatusBadRequest)
+ return
+ }
+ _, isRouter := r.peerRouters.Load(p)
+ view := peerView{
+ PeerID: p.String(),
+ Connectedness: r.Host.Network().Connectedness(p).String(),
+ Authenticated: r.isPeerAuthenticated(p),
+ PeerRouter: isRouter,
+ Connections: []string{},
+ Addrs: []string{},
+ }
+ for _, c := range r.Host.Network().ConnsToPeer(p) {
+ dir := "inbound"
+ if c.Stat().Direction == network.DirOutbound {
+ dir = "outbound"
+ }
+ view.Connections = append(view.Connections, dir+" "+c.RemoteMultiaddr().String())
+ }
+ for _, a := range r.Host.Peerstore().Addrs(p) {
+ view.Addrs = append(view.Addrs, a.String())
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(view)
+}
diff --git a/internal/router/metrics.go b/internal/router/metrics.go
index 0f1e59a9..42bc1f7f 100644
--- a/internal/router/metrics.go
+++ b/internal/router/metrics.go
@@ -113,6 +113,7 @@ type routerStateCollector struct {
readyDesc *prometheus.Desc
drainingDesc *prometheus.Desc
authenticatedDesc *prometheus.Desc
+ peerRoutersDesc *prometheus.Desc
connectedDesc *prometheus.Desc
connectionsDesc *prometheus.Desc
watermarkDesc *prometheus.Desc
@@ -141,6 +142,10 @@ func newRouterStateCollector(r *Router) *routerStateCollector {
"agentmesh_router_authenticated_peers",
"Connected peers that have completed the mesh authentication handshake",
nil, nil),
+ peerRoutersDesc: prometheus.NewDesc(
+ "agentmesh_router_peer_routers",
+ "Peer routers known, from the control plane's list or a handshake with the router role: all of them, and those currently authenticated",
+ []string{"state"}, nil),
connectedDesc: prometheus.NewDesc(
"agentmesh_router_connected_peers",
"Peers with an open libp2p connection, authenticated or not",
@@ -174,7 +179,7 @@ func newRouterStateCollector(r *Router) *routerStateCollector {
func (c *routerStateCollector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
- c.infoDesc, c.readyDesc, c.drainingDesc, c.authenticatedDesc, c.connectedDesc, c.connectionsDesc, c.watermarkDesc, c.relayLimitDesc,
+ c.infoDesc, c.readyDesc, c.drainingDesc, c.authenticatedDesc, c.peerRoutersDesc, c.connectedDesc, c.connectionsDesc, c.watermarkDesc, c.relayLimitDesc,
c.bannedDesc, c.dhtDesc, c.biscuitExpiryDesc, connsPerSourceIPDesc,
} {
ch <- d
@@ -203,6 +208,9 @@ func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.readyDesc, prometheus.GaugeValue, 1)
ch <- prometheus.MustNewConstMetric(c.infoDesc, prometheus.GaugeValue, 1, r.Host.ID().String())
ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(syncMapLen(&r.authenticatedPeers)))
+ listed, federated := r.peerRouterCounts()
+ ch <- prometheus.MustNewConstMetric(c.peerRoutersDesc, prometheus.GaugeValue, float64(listed), "listed")
+ ch <- prometheus.MustNewConstMetric(c.peerRoutersDesc, prometheus.GaugeValue, float64(federated), "authenticated")
ch <- prometheus.MustNewConstMetric(c.connectedDesc, prometheus.GaugeValue, float64(len(r.Host.Network().Peers())))
var inbound, outbound int
for _, conn := range r.Host.Network().Conns() {
@@ -234,7 +242,8 @@ func syncMapLen(m *sync.Map) int {
return n
}
-// serveMetrics opens the operator listener: /metrics, /healthz and /readyz.
+// serveMetrics opens the operator listener: /metrics, /healthz, /readyz and
+// /debug/peer, the router's view of one peer.
// It is off unless an address is configured, and never shares a port with the
// libp2p listeners, so it can stay cluster-internal while the mesh port is
// public. libp2p's own metrics land in the default registry, so exposing it
@@ -263,6 +272,7 @@ func (r *Router) serveMetrics(addr string) error {
}
w.WriteHeader(http.StatusOK)
})
+ mux.HandleFunc("GET /debug/peer", r.handleDebugPeer)
r.metricsServer = &http.Server{
Handler: mux,
diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.go
index 6b5ae0d0..2f98750a 100644
--- a/internal/router/metrics_test.go
+++ b/internal/router/metrics_test.go
@@ -16,6 +16,7 @@ package router
import (
"context"
+ "encoding/json"
"io"
"net/http"
"strconv"
@@ -86,6 +87,9 @@ func TestRouterMetricsListener(t *testing.T) {
r.authenticatedPeers.Store(peer.ID("peer-a"), true)
r.authenticatedPeers.Store(peer.ID("peer-b"), true)
r.bannedPeers.Store(peer.ID("peer-x"), true)
+ // Two peer routers listed, of which one is authenticated.
+ r.peerRouters.Store(peer.ID("peer-a"), true)
+ r.peerRouters.Store(peer.ID("peer-r"), true)
expiry := time.Now().Add(time.Hour).Truncate(time.Second)
r.keysMu.Lock()
r.credential.expiration = expiry
@@ -110,6 +114,8 @@ func TestRouterMetricsListener(t *testing.T) {
`agentmesh_router_relay_limit{limit="circuits_per_peer"} 1024`,
"agentmesh_router_inbound_connections_refused_total",
"agentmesh_router_authenticated_peers 2",
+ `agentmesh_router_peer_routers{state="listed"} 2`,
+ `agentmesh_router_peer_routers{state="authenticated"} 1`,
"agentmesh_router_banned_peers 1",
"agentmesh_router_connected_peers 0",
"agentmesh_router_dht_routing_table_size 0",
@@ -126,6 +132,32 @@ func TestRouterMetricsListener(t *testing.T) {
}
}
+ // /debug/peer is the router's view of one peer: its standing here and
+ // the addresses the router would hand out for it.
+ other, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = other.Close() }()
+ if err := other.Connect(ctx, peer.AddrInfo{ID: host.ID(), Addrs: host.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+ r.authenticatedPeers.Store(other.ID(), time.Now().Add(time.Hour))
+ code, body = getBody(t, base+"/debug/peer?peer_id="+other.ID().String())
+ if code != http.StatusOK {
+ t.Fatalf("/debug/peer = %d: %s", code, body)
+ }
+ var view peerView
+ if err := json.Unmarshal([]byte(body), &view); err != nil {
+ t.Fatalf("/debug/peer: %v: %s", err, body)
+ }
+ if view.Connectedness != "Connected" || !view.Authenticated || view.PeerRouter || len(view.Connections) != 1 || len(view.Addrs) == 0 {
+ t.Errorf("/debug/peer for a connected, authenticated member: %+v", view)
+ }
+ if code, _ := getBody(t, base+"/debug/peer?peer_id=nonsense"); code != http.StatusBadRequest {
+ t.Errorf("/debug/peer with a bad peer id = %d, want 400", code)
+ }
+
if err := r.Close(); err != nil {
t.Fatal(err)
}
diff --git a/internal/router/router.go b/internal/router/router.go
index 794fa17f..9c83059f 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -311,7 +311,9 @@ type Router struct {
// identityLoggedAt is when the "Router Online" line was last written.
identityLoggedAt atomic.Pointer[time.Time]
- authFailureLog perPeerLogLimiter
+ // leasedAddrs is the address list of the last accepted lease, joined.
+ leasedAddrs atomic.Pointer[string]
+ authFailureLog perPeerLogLimiter
}
// NewRouter initializes the router.
@@ -1151,12 +1153,29 @@ func (r *Router) sendLease(ctx context.Context, ttl *time.Duration, attempts int
} else {
logger.Debugf("Lease renewed successfully. Expires at: %s", leaseResp.GetExpireTime().AsTime().Format(time.RFC3339))
leaseRenewalsTotal.WithLabelValues(leaseOK).Inc()
+ r.logLeasedAddrs(addrs)
r.logIdentityPeriodically()
}
return
}
}
+// logLeasedAddrs writes the addresses the control plane now lists for this
+// router each time they differ from the last lease: members and peer routers
+// reach the router only through these, and a lease without any leaves it
+// listed but unreachable.
+func (r *Router) logLeasedAddrs(addrs []string) {
+ joined := strings.Join(addrs, ",")
+ if last := r.leasedAddrs.Swap(&joined); last != nil && *last == joined {
+ return
+ }
+ if len(addrs) == 0 {
+ logger.Warn("Lease advertises no addresses: the control plane lists this router but nothing can reach it")
+ return
+ }
+ logger.Infof("Lease advertises %d address(es): %v", len(addrs), addrs)
+}
+
// identityLogInterval is how often the router restates who it is. Container
// logs rotate, and the startup line is the first to go; an operator reading
// the log should still find the peer ID within the last hour of it.
@@ -1277,37 +1296,66 @@ func (r *Router) connectBootstrapRouters() {
for _, resolved := range resolvedAddrs {
pi, err := peer.AddrInfoFromP2pAddr(resolved)
- if err != nil || pi.ID == r.Host.ID() {
+ if err != nil {
+ logger.Warnf("[Federation] Ignoring router address %s: %v", resolved, err)
+ continue
+ }
+ if pi.ID == r.Host.ID() {
continue
}
r.peerRouters.Store(pi.ID, true)
r.Host.ConnManager().Protect(pi.ID, federationTag)
-
- if len(r.Host.Network().ConnsToPeer(pi.ID)) == 0 {
- logger.Infof("[Federation] Connecting to peer router: %s via %s", pi.ID, resolved)
- // Create a timeout context
- connectCtx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
- if err := r.Host.Connect(connectCtx, *pi); err == nil {
- // Initiate stream to perform mutual auth handshake
- s, err := r.Host.NewStream(connectCtx, pi.ID, api.AuthProtocolID)
- if err == nil {
- if err := r.performMutualAuth(s); err != nil {
- logger.Errorf("[Federation] Mutual auth handshake failed with %s: %v", pi.ID, err)
- _ = s.Reset()
- } else {
- logger.Infof("[Federation] Mutually authenticated with peer router %s", pi.ID)
- _ = s.Close()
- }
- } else {
- logger.Errorf("[Federation] Failed to open auth stream to %s: %v", pi.ID, err)
- }
- } else {
- logger.Errorf("[Federation] Failed to connect to %s: %v", pi.ID, err)
- }
- cancel()
+ if r.isPeerAuthenticated(pi.ID) {
+ continue
}
+ r.authenticatePeerRouter(*pi)
+ }
+ }
+}
+
+// authenticatePeerRouter runs the mutual handshake with a router the control
+// plane lists, over the connection to it if one is open (the DHT commonly
+// opens one first), dialling otherwise. Authentication is what decides, not
+// the connection: a connection the DHT made is not an authenticated one,
+// and an admission that expired is renewed on the next pass.
+func (r *Router) authenticatePeerRouter(pi peer.AddrInfo) {
+ ctx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
+ defer cancel()
+ if len(r.Host.Network().ConnsToPeer(pi.ID)) == 0 {
+ logger.Infof("[Federation] Connecting to peer router: %s via %v", pi.ID, pi.Addrs)
+ if err := r.Host.Connect(ctx, pi); err != nil {
+ logger.Errorf("[Federation] Failed to connect to %s: %v", pi.ID, err)
+ return
}
+ } else {
+ logger.Infof("[Federation] Authenticating with peer router %s over its open connection", pi.ID)
+ }
+ s, err := r.Host.NewStream(ctx, pi.ID, api.AuthProtocolID)
+ if err != nil {
+ logger.Errorf("[Federation] Failed to open auth stream to %s: %v", pi.ID, err)
+ return
+ }
+ if err := r.performMutualAuth(s); err != nil {
+ logger.Errorf("[Federation] Mutual auth handshake failed with %s: %v", pi.ID, err)
+ _ = s.Reset()
+ return
}
+ logger.Infof("[Federation] Mutually authenticated with peer router %s", pi.ID)
+ _ = s.Close()
+}
+
+// peerRouterCounts is how many peer routers this router knows, from the
+// control plane's list or a handshake that proved the router role, and how
+// many of those hold a current authentication.
+func (r *Router) peerRouterCounts() (listed, authenticated int) {
+ r.peerRouters.Range(func(k, _ any) bool {
+ listed++
+ if r.isPeerAuthenticated(k.(peer.ID)) {
+ authenticated++
+ }
+ return true
+ })
+ return listed, authenticated
}
// recoverStreamHandler isolates a panic while processing untrusted peer bytes
@@ -1376,7 +1424,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
}
// Verify Biscuit and record its expiration so relayACL drops the session on time.
- expiry, err := identity.VerifyBiscuitAndGetExpiry(exchange.Biscuit, remotePeer, r.getTrustedPublicKeys(), r.config.BiscuitTimeout)
+ b, verifyingKey, expiry, err := identity.VerifyBiscuitAdmission(exchange.Biscuit, remotePeer, r.getTrustedPublicKeys(), r.config.BiscuitTimeout)
if err != nil {
// A peer stuck on a bad credential retries every few seconds for
// days; one line a minute per peer keeps the log readable and the
@@ -1392,6 +1440,14 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
r.admitPeer(remotePeer, expiry)
authHandshakesTotal.WithLabelValues(handshakeOK).Inc()
logger.Infof("[AuthN] Successfully authenticated peer %s", remotePeer)
+ // A router that proves the router role is a peer router from here on,
+ // not from the next federation pass: it is never a member to shed.
+ if identity.RequireRole(b, verifyingKey, api.RoleRouter, r.config.BiscuitTimeout) == nil {
+ if _, known := r.peerRouters.LoadOrStore(remotePeer, true); !known {
+ logger.Infof("[Federation] Peer router %s authenticated with us", remotePeer)
+ r.Host.ConnManager().Protect(remotePeer, federationTag)
+ }
+ }
// Send mutual response (our biscuit)
r.keysMu.RLock()
diff --git a/internal/router/router_test.go b/internal/router/router_test.go
index 31a6e40c..4f7b2ccd 100644
--- a/internal/router/router_test.go
+++ b/internal/router/router_test.go
@@ -30,6 +30,7 @@ import (
"path/filepath"
"strings"
"sync"
+ "sync/atomic"
"testing"
"time"
@@ -392,8 +393,8 @@ func TestRouterFederation(t *testing.T) {
// Wait for both routers to renew their lease so they are both in control plane DB
time.Sleep(3 * time.Second)
- // Trigger connection sync manually to form the federation link
- r1.connectBootstrapRouters()
+ // Router 2's pass alone forms the link: Router 1 learns Router 2 is a
+ // router from the handshake, not from a pass of its own.
r2.connectBootstrapRouters()
// Wait a moment for stream handshakes to complete
@@ -413,6 +414,15 @@ func TestRouterFederation(t *testing.T) {
if _, authenticated := r2.authenticatedPeers.Load(r1.Host.ID()); !authenticated {
t.Errorf("Router 1 is not authenticated in Router 2's peers")
}
+
+ for name, r := range map[string]*Router{"Router 1": r1, "Router 2": r2} {
+ if listed, authenticated := r.peerRouterCounts(); listed != 1 || authenticated != 1 {
+ t.Errorf("%s knows %d peer router(s), %d authenticated; want 1 and 1", name, listed, authenticated)
+ }
+ if members := r.members(); len(members) != 0 {
+ t.Errorf("%s counts %v as members; a peer router is not one", name, members)
+ }
+ }
}
func TestRouterProactiveTokenRefresh(t *testing.T) {
@@ -1043,6 +1053,76 @@ func TestPerformMutualAuth(t *testing.T) {
}
}
+// A connection to a peer router is commonly open before the federation pass
+// reaches it (the DHT dials routers it learns of), and an admission expires.
+// Neither leaves the pair unauthenticated: the pass authenticates whenever
+// the peer is not currently admitted, over whatever connection is open.
+func TestFederationAuthenticatesListedRoutersNotCurrentlyAdmitted(t *testing.T) {
+ pub, priv, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ peerRouter, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = peerRouter.Close() }()
+ peerBiscuit, err := identity.MintBootstrapBiscuitToken(priv, peerRouter.ID(), api.RoleRouter, time.Now().Add(time.Hour), nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var handshakes atomic.Int32
+ peerRouter.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {
+ defer func() { _ = s.Close() }()
+ reader := msgio.NewVarintReaderSize(s, 1024*64)
+ msg, err := reader.ReadMsg()
+ if err != nil {
+ return
+ }
+ reader.ReleaseMsg(msg)
+ handshakes.Add(1)
+ respBytes, _ := proto.Marshal(&api.AuthResponse{Success: true, Biscuit: peerBiscuit})
+ _ = msgio.NewVarintWriter(s).WriteMsg(respBytes)
+ })
+
+ r := newGoAwayRouter(t, 100)
+ r.ctx = context.Background()
+ r.credential = credential{biscuit: []byte("router-biscuit")}
+ r.trustedPublicKeys = []ed25519.PublicKey{pub}
+ r.config.BiscuitTimeout = time.Second
+ r.config.RequiredRole = api.RoleRouter
+ pi := peer.AddrInfo{ID: peerRouter.ID(), Addrs: peerRouter.Addrs()}
+
+ // The DHT got there first.
+ if err := r.Host.Connect(context.Background(), pi); err != nil {
+ t.Fatal(err)
+ }
+ r.peerRouters.Store(pi.ID, true)
+ if listed, authenticated := r.peerRouterCounts(); listed != 1 || authenticated != 0 {
+ t.Fatalf("before the pass: listed %d, authenticated %d; want 1, 0", listed, authenticated)
+ }
+ r.authenticatePeerRouter(pi)
+ if !r.isPeerAuthenticated(pi.ID) {
+ t.Fatal("peer router connected by the DHT was not authenticated")
+ }
+ if listed, authenticated := r.peerRouterCounts(); listed != 1 || authenticated != 1 {
+ t.Fatalf("after the pass: listed %d, authenticated %d; want 1, 1", listed, authenticated)
+ }
+ if got := handshakes.Load(); got != 1 {
+ t.Fatalf("handshakes with the peer router: %d, want 1", got)
+ }
+
+ // The admission lapsed; it is renewed, not left to the connection.
+ r.authenticatedPeers.Store(pi.ID, time.Now().Add(-time.Second))
+ if _, authenticated := r.peerRouterCounts(); authenticated != 0 {
+ t.Fatal("an expired admission counts as authenticated")
+ }
+ r.authenticatePeerRouter(pi)
+ if !r.isPeerAuthenticated(pi.ID) || handshakes.Load() != 2 {
+ t.Fatalf("expired admission not renewed: authenticated %v, handshakes %d", r.isPeerAuthenticated(pi.ID), handshakes.Load())
+ }
+}
+
// The router's /mesh/auth handler is reachable by any internet peer. An idle
// stream must be closed on the router's schedule, and a peer handshaking in a
// tight loop must be cut off before every frame is verified.
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index cf97c549..a7cb5fdc 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -50,7 +50,7 @@ replica count.
| `--relay-max-circuits` | `1024` | Relayed connections one peer may hold at once through this router, as caller or as destination. A service that many members call at once needs this many open circuits. |
| `--shutdown-lease-ttl` | `30s` | Sent in the router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. The default covers a pod that restarts in place. A router that is being removed for good can send `1s`. `0` sends nothing and the lease expires on the control plane's schedule. |
| `--auth-deadline` | `1m` | How long an inbound connection may stay open without its peer passing the handshake. A member handshakes the moment it connects; a member attached elsewhere whose DHT client came to query is done in less, as is the control plane's event publisher, which presents a credential of its own. What is left at the deadline is closed, so the connections a router holds are its members' and the watermarks mean members. `0` keeps every connection. |
-| `--metrics-addr` | off | Serve `/metrics`, `/healthz` and `/readyz` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. |
+| `--metrics-addr` | off | Serve `/metrics`, `/healthz`, `/readyz` and `/debug/peer` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. |
| `--log-level` | `info` | `debug`, `info`, `warn`, `error`. `LOG_FORMAT=json` selects JSON output. |
## What it does at run time
@@ -114,6 +114,7 @@ The metrics address exposes, next to the Go runtime metrics:
| `agentmesh_router_conns_per_source_ip_limit` | The per-address cap in force. |
| `agentmesh_router_relay_limit{limit}` | The relay budget in force: `reservations`, `reservations_per_ip` and `circuits_per_peer`. |
| `agentmesh_router_connected_peers`, `agentmesh_router_authenticated_peers`, `agentmesh_router_banned_peers` | Peers in each state. |
+| `agentmesh_router_peer_routers{state}` | Other routers this one knows, from the control plane's list or a handshake that proved the router role: `listed`, and `authenticated` for those it currently holds a mutual authentication with. The two differ while a federation pass is pending or a peer router is down. |
| `agentmesh_router_dht_routing_table_size` | DHT routing table size. |
| `agentmesh_router_auth_handshakes_total{result}`, `agentmesh_router_lease_renewals_total{result}` | Handshakes and lease renewals by outcome. |
| `agentmesh_router_goaway_sent_total{reason,result}` | Members sent away, by reason (`DRAINING`, `OVERLOADED`) and outcome: `ok` when the member took the message, `unsupported` when it does not speak the protocol and was closed instead, `failed` when the stream broke. A rising `OVERLOADED` count on a router that is not draining is a fleet that has outgrown its routers. |
@@ -126,3 +127,10 @@ approaches the `high` watermark or open relay reservations approach
`agentmesh_router_relay_limit{limit="reservations"}` on every router that is not
draining; both are the inputs for an autoscaler. `/healthz` and `/readyz`
on the same address are the probes to use in a pod spec.
+
+`GET /debug/peer?peer_id=<id>` on the same address is the router's view of
+one peer: its connectedness, whether it is authenticated here and whether it
+is a peer router, its open connections, and the addresses the router holds
+for it, which are the ones it hands out in a DHT answer. It is a diagnostic
+for an operator reading why a member cannot reach another, and its shape is
+not versioned.
diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.go
index 0f345660..365dc447 100644
--- a/tests/integration/router_attachment_test.go
+++ b/tests/integration/router_attachment_test.go
@@ -340,10 +340,24 @@ bindings:
"--insecure-skip-tls-verify",
"--mesh-reconnect-interval", "500ms",
)
+ cpLog := filepath.Join(m.tmpDir, "control-plane.log")
+ cpLogFile, err := os.Create(cpLog)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cpCmd.Stdout, cpCmd.Stderr = cpLogFile, cpLogFile
if err := cpCmd.Start(); err != nil {
t.Fatalf("start control plane: %v", err)
}
- t.Cleanup(func() { _ = cpCmd.Process.Kill(); _ = cpCmd.Wait() })
+ t.Cleanup(func() {
+ _ = cpCmd.Process.Kill()
+ _ = cpCmd.Wait()
+ _ = cpLogFile.Close()
+ if t.Failed() {
+ out, _ := os.ReadFile(cpLog)
+ t.Logf("--- control plane ---\n%s", out)
+ }
+ })
waitForControlPlane(t, m.cpPort)
injectPolicyYAML(t, m.cpPort, testAdminToken, policyFile)
return m
@@ -526,9 +540,13 @@ func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {
for _, s := range sites {
routers[s] = m.startRouter(t, "router-"+s, "site="+s)
}
- // Federation on the first pass: every router finds the ones before it.
- waitForEqual(t, "routers connected to router c", func() []string { return routersConnectedTo(t, m.cpPort, routers["c"].peerID) },
- peerIDStrings(routers["a"].peerID, routers["b"].peerID), 10*time.Second)
+ // Federation on the first pass: every router finds the ones before it,
+ // and authenticates them whether or not the DHT connected them first.
+ for _, s := range sites {
+ waitForEqual(t, "peer routers router "+s+" authenticated", func() []string {
+ return []string{fmt.Sprint(routers[s].gauge(t, `agentmesh_router_peer_routers{state="authenticated"}`))}
+ }, []string{"2"}, 10*time.Second)
+ }
nodes := map[string]*backgroundNode{}
for _, s := range sites {
@@ -574,6 +592,23 @@ func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {
}
}
if t.Failed() {
+ known := map[string]string{}
+ for _, s := range sites {
+ known[routers[s].peerID.String()] = "router " + s
+ known[nodes[s].peerID.String()] = "node " + s
+ }
+ for _, lease := range fetchAdminStatus(t, m.cpPort, testAdminToken).GetActiveRouters() {
+ var peers []string
+ for _, p := range lease.GetConnectedPeers() {
+ name := known[p]
+ if name == "" {
+ name = "other " + p[len(p)-6:]
+ }
+ peers = append(peers, name)
+ }
+ slices.Sort(peers)
+ t.Logf("%s leases %d address(es), connected to: %v", known[lease.GetPeerId()], len(lease.GetAddresses()), peers)
+ }
for _, s := range sites {
out := nodes[s].log()
if i := strings.LastIndex(out, "[AuthN] Successfully authenticated"); i > 0 {