Skip to content

Agent Mesh

node: handshake with a transit router the catalog does not list

Antonio Ojeab221cee149a3
diff --git a/internal/node/mcp.go b/internal/node/mcp.go
index db9ff545..ae22dc9c 100644
--- a/internal/node/mcp.go
+++ b/internal/node/mcp.go
@@ -590,6 +590,24 @@ func circuitRelay(ma multiaddr.Multiaddr) peer.ID {
return relayID
}
+// relayTransportAddr is the relay's own address carried in a circuit
+// address, or nil for a bare /p2p/<relay>/p2p-circuit.
+func relayTransportAddr(ma multiaddr.Multiaddr) multiaddr.Multiaddr {
+ parts := multiaddr.Split(ma)
+ if len(parts) < 3 {
+ return nil
+ }
+ var via multiaddr.Multiaddr
+ for _, c := range parts[:len(parts)-2] {
+ via = via.AppendComponent(&c)
+ }
+ return via
+}
+
+// transitCatalogRefreshInterval bounds how often a relay the catalog does
+// not list makes the node re-read /info.
+const transitCatalogRefreshInterval = 10 * time.Second
+
// selectTransit decides which of a peer's circuit addresses to dial and
// makes sure the node is admitted at the router behind them. A router
// relays only between peers it has admitted, and a node keeps sessions
@@ -600,30 +618,69 @@ func circuitRelay(ma multiaddr.Multiaddr) peer.ID {
// at start, before the circuit is asked for. Circuits through the peer's
// other routers are dropped: each would be a handshake and a session
// there, and a service the whole mesh calls would hold every caller at
-// every one of its routers. A relay that is not a router (a node with
-// --enable-relay) is kept and left alone; it admits what it has handshaken
-// with on its own terms. Direct addresses always pass.
+// every one of its routers.
+//
+// A relay the catalog does not list is first looked for in a fresh /info:
+// the catalog is read at start, a router that was restarting then was not
+// in it, and a member that started during a rollout would otherwise never
+// transit that router until the next sync. A relay still unlisted after
+// that is handshaken with all the same, after the listed routers: a router
+// whose lease the control plane does not carry admits the node like any
+// other, and a relay that is not a router (a node with --enable-relay)
+// admits what it has handshaken with on its own terms and is kept whatever
+// the handshake said. Direct addresses always pass.
func (n *AgentMeshNode) selectTransit(ctx context.Context, addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr {
- routers := map[peer.ID]routerCandidate{}
- for _, c := range n.routerCandidates() {
- routers[c.ID] = c
- }
- var viaRouters []peer.ID
- seen := map[peer.ID]bool{}
+ var relays []peer.ID
+ relayAddrs := map[peer.ID][]multiaddr.Multiaddr{}
for _, ma := range addrs {
relay := circuitRelay(ma)
- if _, isRouter := routers[relay]; relay == "" || !isRouter || seen[relay] {
+ if relay == "" {
continue
}
- seen[relay] = true
- viaRouters = append(viaRouters, relay)
+ if _, seen := relayAddrs[relay]; !seen {
+ relays = append(relays, relay)
+ }
+ if via := relayTransportAddr(ma); via != nil {
+ relayAddrs[relay] = append(relayAddrs[relay], via)
+ } else if _, seen := relayAddrs[relay]; !seen {
+ relayAddrs[relay] = nil
+ }
}
- if len(viaRouters) == 0 {
+ if len(relays) == 0 {
return addrs
}
+ routers := map[peer.ID]routerCandidate{}
+ for _, c := range n.routerCandidates() {
+ routers[c.ID] = c
+ }
+ var unlisted []peer.ID
+ for _, r := range relays {
+ if _, isRouter := routers[r]; !isRouter {
+ unlisted = append(unlisted, r)
+ }
+ }
+ if len(unlisted) > 0 && n.refreshCatalogForTransit(ctx) {
+ routers = map[peer.ID]routerCandidate{}
+ for _, c := range n.routerCandidates() {
+ routers[c.ID] = c
+ }
+ unlisted = unlisted[:0]
+ for _, r := range relays {
+ if _, isRouter := routers[r]; !isRouter {
+ unlisted = append(unlisted, r)
+ }
+ }
+ }
+ var viaRouters []peer.ID
+ for _, r := range relays {
+ if _, isRouter := routers[r]; isRouter {
+ viaRouters = append(viaRouters, r)
+ }
+ }
+
usable := map[peer.ID]bool{}
- for _, r := range viaRouters {
+ for _, r := range relays {
if n.isAuthenticatedAndConnected(r) {
usable[r] = true
}
@@ -637,12 +694,20 @@ func (n *AgentMeshNode) selectTransit(ctx context.Context, addrs []multiaddr.Mul
logger.Debugf("[AuthN] Cannot authenticate to a transit router: %v", err)
return addrs
}
- for _, r := range viaRouters {
- router := peer.AddrInfo{ID: r, Addrs: n.Host.Peerstore().Addrs(r)}
+ for _, r := range append(viaRouters, unlisted...) {
+ _, listed := routers[r]
+ kind := transitListed
+ if !listed {
+ kind = transitUnlisted
+ logger.Infof("[AuthN] Relay %s is not a router the control plane lists; handshaking with it for the circuit", r)
+ }
+ router := peer.AddrInfo{ID: r, Addrs: append(n.Host.Peerstore().Addrs(r), relayAddrs[r]...)}
if err := n.authRouter(ctx, router, biscuitBytes); err != nil {
+ transitHandshakesTotal.WithLabelValues(kind, "failed").Inc()
logger.Warnf("[AuthN] Transit router %s did not admit this node: %v", r, err)
continue
}
+ transitHandshakesTotal.WithLabelValues(kind, "ok").Inc()
logger.Debugf("[AuthN] Authenticated to transit router %s", r)
usable[r] = true
break
@@ -662,3 +727,25 @@ func (n *AgentMeshNode) selectTransit(ctx context.Context, addrs []multiaddr.Mul
}
return kept
}
+
+// refreshCatalogForTransit re-reads the routers from the control plane for
+// a relay the catalog does not list, at most once per
+// transitCatalogRefreshInterval. It reports whether the catalog was re-read.
+func (n *AgentMeshNode) refreshCatalogForTransit(ctx context.Context) bool {
+ n.mu.Lock()
+ due := time.Since(n.transitCatalogRefreshedAt) >= transitCatalogRefreshInterval
+ if due {
+ n.transitCatalogRefreshedAt = time.Now()
+ }
+ n.mu.Unlock()
+ if !due {
+ return false
+ }
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+ if err := n.refreshRouterCatalog(ctx); err != nil {
+ logger.Debugf("[AuthN] Routers not re-read from the control plane for a transit relay: %v", err)
+ return false
+ }
+ return true
+}
diff --git a/internal/node/node.go b/internal/node/node.go
index e1334b8a..3caf6228 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -169,23 +169,26 @@ type AgentMeshNode struct {
// Options.Routers of them it keeps a session with; shunned are the ones
// it stays off until the time given, having been sent away or having
// not come back. connMgr protects the attached ones from being trimmed.
- routerCatalog []routerCandidate
- attached map[peer.ID]bool
- shunned map[peer.ID]time.Time
- connMgr *connmgr.BasicConnMgr
- peerLastEventTime map[string]int64
- mu sync.Mutex
- nodeConfig *NodeConfigComplete
- revokedPeers *lru.Cache[string, int64]
- revokedTokens *lru.Cache[string, time.Time]
- delegatedBiscuitCache *lru.Cache[string, *api.TokenExchangeResponse]
- stsTokenCache *lru.Cache[string, *api.STSTokenResponse]
- peerLabelGate *lru.Cache[string, time.Time]
- authPeers sync.Map
- trustedKeys []TrustedKey
- keysMu sync.RWMutex
- MeshPolicyRules []biscuit.Rule
- MeshPolicyMu sync.RWMutex
+ routerCatalog []routerCandidate
+ // transitCatalogRefreshedAt is when a relay the catalog did not list
+ // last made the node re-read /info (selectTransit).
+ transitCatalogRefreshedAt time.Time
+ attached map[peer.ID]bool
+ shunned map[peer.ID]time.Time
+ connMgr *connmgr.BasicConnMgr
+ peerLastEventTime map[string]int64
+ mu sync.Mutex
+ nodeConfig *NodeConfigComplete
+ revokedPeers *lru.Cache[string, int64]
+ revokedTokens *lru.Cache[string, time.Time]
+ delegatedBiscuitCache *lru.Cache[string, *api.TokenExchangeResponse]
+ stsTokenCache *lru.Cache[string, *api.STSTokenResponse]
+ peerLabelGate *lru.Cache[string, time.Time]
+ authPeers sync.Map
+ trustedKeys []TrustedKey
+ keysMu sync.RWMutex
+ MeshPolicyRules []biscuit.Rule
+ MeshPolicyMu sync.RWMutex
// pendingEgress holds assignments that arrived before Start created the
// service registry (SyncControlPlane runs first); Start applies them.
pendingEgress []*api.EgressDestination
@@ -1153,6 +1156,9 @@ func (n *AgentMeshNode) redialRouter(ctx context.Context, router peer.ID) {
}
func (n *AgentMeshNode) loadIdentityForAuth() ([]byte, error) {
+ if n.Store == nil {
+ return nil, fmt.Errorf("%w: no identity store", ErrFatalAuth)
+ }
biscuitBytes, err := n.Store.LoadIdentity()
if err != nil {
return nil, fmt.Errorf("%w: failed to load identity from store: %w", ErrFatalAuth, err)
diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.go
index 119e7689..aab5824d 100644
--- a/internal/node/node_metrics.go
+++ b/internal/node/node_metrics.go
@@ -41,8 +41,31 @@ func init() {
for _, reason := range []api.RouterGoAway_Reason{api.RouterGoAway_DRAINING, api.RouterGoAway_OVERLOADED} {
goAwayReceivedTotal.WithLabelValues(reason.String())
}
+ for _, router := range []string{transitListed, transitUnlisted} {
+ for _, result := range []string{"ok", "failed"} {
+ transitHandshakesTotal.WithLabelValues(router, result)
+ }
+ }
}
+// The router a transit handshake is with: one the control plane's catalog
+// lists, or a relay found only in a peer's record.
+const (
+ transitListed = "listed"
+ transitUnlisted = "unlisted"
+)
+
+// transitHandshakesTotal counts the handshakes a call made with a router it
+// did not hold, to reach a peer through it; the unlisted ones are routers
+// the catalog missed, which a rising count during a rollout explains.
+var transitHandshakesTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_node_transit_handshakes_total",
+ Help: "Handshakes with a router this node did not hold, run to reach a peer through it, by whether the control plane listed the router and by result",
+ },
+ []string{"router", "result"},
+)
+
// nodeStateCollector exports the node's own view of its mesh membership: the
// same figures GET /debug/mesh-info and /debug/token-info answer on demand,
// as gauges a scraper can keep. A node that runs unattended, such as a
diff --git a/internal/node/router_auth_dedupe_test.go b/internal/node/router_auth_dedupe_test.go
index ee886281..d7aa2676 100644
--- a/internal/node/router_auth_dedupe_test.go
+++ b/internal/node/router_auth_dedupe_test.go
@@ -608,3 +608,44 @@ func TestRouterDropIsRedialed(t *testing.T) {
t.Fatalf("handshakes after the second drop = %d, want 3", got)
}
}
+
+// A peer's record names a circuit through a router the node's catalog does
+// not list: the node read the catalog while that router was restarting, or
+// the router's lease is not carried right now. The call must handshake with
+// that router before asking it for the circuit, as it would with a listed
+// one; without a control plane to re-read, the record is all it has.
+func TestSelectTransitHandshakesWithARouterTheCatalogDoesNotList(t *testing.T) {
+ known := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ unlisted := newFakeRouterWithKey(t, known.cpPub, known.cpPriv, "/ip4/127.0.0.1/tcp/0")
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ // The node holds the known router; the store carries no control plane
+ // URL, so a re-read of the catalog is not possible here.
+ node := known.startNode(t, ctx, known.routerAddrs)
+ if node.isAuthenticatedAndConnected(unlisted.h.ID()) {
+ t.Fatal("node already admitted at the router it was never told about")
+ }
+
+ direct := multiaddr.StringCast("/ip4/192.0.2.9/tcp/5002")
+ circuit := unlisted.routerAddrs[0].Encapsulate(multiaddr.StringCast("/p2p-circuit"))
+ kept := node.selectTransit(ctx, []multiaddr.Multiaddr{direct, circuit})
+
+ if got := unlisted.handshakes.Load(); got != 1 {
+ t.Fatalf("unlisted router saw %d handshakes, want 1", got)
+ }
+ if !node.isAuthenticatedAndConnected(unlisted.h.ID()) {
+ t.Fatal("node not admitted at the unlisted router after the handshake")
+ }
+ var got []string
+ for _, a := range kept {
+ got = append(got, a.String())
+ }
+ if len(kept) != 2 || kept[0].String() != direct.String() || kept[1].String() != circuit.String() {
+ t.Fatalf("addresses kept: %v, want the direct address and the circuit through the unlisted router", got)
+ }
+ // Admitted now: a second selection runs no handshake.
+ node.selectTransit(ctx, []multiaddr.Multiaddr{circuit})
+ if got := unlisted.handshakes.Load(); got != 1 {
+ t.Fatalf("unlisted router saw %d handshakes after the node was admitted, want still 1", got)
+ }
+}
diff --git a/internal/node/routers.go b/internal/node/routers.go
index 58236ce5..7f1ba033 100644
--- a/internal/node/routers.go
+++ b/internal/node/routers.go
@@ -173,6 +173,9 @@ func (n *AgentMeshNode) routerCandidates() []routerCandidate {
// refreshRouterCatalog re-reads /info. Best effort: a node started from its
// stored addresses with no control plane in reach keeps what it has.
func (n *AgentMeshNode) refreshRouterCatalog(ctx context.Context) error {
+ if n.Store == nil {
+ return errors.New("no store")
+ }
controlPlaneURL, err := n.Store.LoadControlPlaneURL()
if err != nil || controlPlaneURL == "" {
return errors.New("no control plane URL in store")
diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.md
index fde857a6..3c31ecfe 100644
--- a/site/content/docs/guides/scalability.md
+++ b/site/content/docs/guides/scalability.md
@@ -225,6 +225,7 @@ The router and the control plane serve Prometheus metrics on
| `agentmesh_router_goaway_sent_total{reason="OVERLOADED"}` | A router at its high mark sent members to the others. Rising on a router that is not draining, the fleet has outgrown its routers. |
| `agentmesh_router_inbound_connections_refused_total` | Members behind one address have hit `--conns-per-source-ip`. |
| `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `scope_limit` more callers at once than the node's connection budget. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. |
+| `agentmesh_node_transit_handshakes_total{router,result}` | On a caller: handshakes with routers it did not hold, run to reach a peer through them. `router="unlisted"` is a router the node's catalog missed, which a member that started during a router rollout produces until its next sync; `result="failed"` rising is a router that refuses the node or is gone while peers still announce circuits through it. |
| `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. |
| `libp2p_relaysvc_connection_rejections_total{reason="resource limit exceeded"}` | A service has more callers than `--relay-max-circuits`. |
| `libp2p_rcmgr_blocked_resources` | libp2p refused a connection or stream on a scope. On a router this should stay flat. |
diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.go
index 365dc447..59293f5d 100644
--- a/tests/integration/router_attachment_test.go
+++ b/tests/integration/router_attachment_test.go
@@ -232,6 +232,7 @@ func circuitRelaysInRecord(t *testing.T, reader *backgroundNode, target peer.ID)
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
// Not in the DHT yet.
+ t.Logf("peer-addrs for %s: %d %s", target, resp.StatusCode, strings.TrimSpace(string(body)))
return nil
}
var info struct {
@@ -718,6 +719,60 @@ func TestCallerTransitsTheRouterItShares(t *testing.T) {
}
}
+// TestCallerTransitsARouterItsCatalogMissed is the member that started
+// while a router was away: its catalog, read from /info once at start, does
+// not list a router that joined after, and a peer on that router announces
+// circuits only through it. A circuit through a router the caller has not
+// handshaken with is refused by the relay, so the caller must learn the
+// router from the record and admit itself there before it asks.
+func TestCallerTransitsARouterItsCatalogMissed(t *testing.T) {
+ m := startAttachmentMesh(t)
+ routerA := m.startRouter(t, "router-a", "site=a")
+ common := append(m.nodeArgs(), "--monitor-bootstrap", "500ms", "--monitor-interval", "500ms")
+ // One router wanted and one matching, so the caller is never short and
+ // never re-reads /info on its own.
+ caller := launchNode(t, m.nodeBin, m.nodeEnv("caller"), filepath.Join(m.tmpDir, "caller"), append(slices.Clone(common),
+ "--api-token-path", tokenPath(t, "token-c"),
+ "--router-selector", "site=a",
+ "--routers", "1",
+ )...)
+ caller.waitForAPI(t)
+ t.Cleanup(func() {
+ if t.Failed() {
+ t.Logf("--- caller ---\n%s", caller.log())
+ }
+ })
+ waitForEqual(t, "routers attached by the caller", func() []string { return attachedRouters(t, caller) }, peerIDStrings(routerA.peerID), 10*time.Second)
+
+ // Router b arrives after the caller; the provider lives only there.
+ routerB := m.startRouter(t, "router-b", "site=b")
+ waitForEqual(t, "peer routers router b authenticated", func() []string {
+ return []string{fmt.Sprint(routerB.gauge(t, `agentmesh_router_peer_routers{state="authenticated"}`))}
+ }, []string{"1"}, 10*time.Second)
+ mcpServer := httptest.NewServer(newBoundaryMCPHandler(t))
+ t.Cleanup(mcpServer.Close)
+ provider := launchNode(t, m.nodeBin, m.nodeEnv("provider"), filepath.Join(m.tmpDir, "provider"), append(slices.Clone(common),
+ "--api-token-path", tokenPath(t, "token-p"),
+ "--router-selector", "site=b",
+ "--config", writeNodeConfig(t, m.tmpDir, nil, svcDecl{Type: "mcp", Name: "late-tool", TargetURL: mcpServer.URL}),
+ )...)
+ provider.waitForAPI(t)
+ waitForEqual(t, "routers attached by the provider", func() []string { return attachedRouters(t, provider) }, peerIDStrings(routerB.peerID), 10*time.Second)
+ waitForEqual(t, "routers the provider announces circuits through", func() []string { return announcedCircuitRelays(t, provider) },
+ peerIDStrings(routerB.peerID), 15*time.Second)
+ // A peer lookup by ID has to connect the peer, and the only path is the
+ // circuit through router b, so the record is not readable from here
+ // before the handshake there; discovery by service name needs no dial.
+ callAcrossMesh(t, caller, "token-c", provider, "late-tool")
+
+ // Router b admitted the provider, router a, the control plane's publisher
+ // and, for the call, the caller.
+ if got := routerB.gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {
+ t.Errorf("router b authenticated %v peers, want 4: the caller must have handshaken there before asking for the circuit", got)
+ }
+ t.Logf("--- caller ---\n%s", caller.log())
+}
+
// TestRouterKeepsOnlyAdmittedConnections pins what a router holds: a
// connection whose peer does not pass the handshake within --auth-deadline
// is closed, and the control plane's event publisher, which presents a