Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: handshake with a transit router the catalog does not list
Antonio Ojeab221cee149a3
diff --git a/internal/node/mcp.go b/internal/node/mcp.goindex db9ff545..ae22dc9c 100644--- a/internal/node/mcp.go+++ b/internal/node/mcp.go@@ -590,6 +590,24 @@ func circuitRelay(ma multiaddr.Multiaddr) peer.ID {return relayID}+// relayTransportAddr is the relay's own address carried in a circuit+// address, or nil for a bare /p2p/<relay>/p2p-circuit.+func relayTransportAddr(ma multiaddr.Multiaddr) multiaddr.Multiaddr {+ parts := multiaddr.Split(ma)+ if len(parts) < 3 {+ return nil+ }+ var via multiaddr.Multiaddr+ for _, c := range parts[:len(parts)-2] {+ via = via.AppendComponent(&c)+ }+ return via+}++// transitCatalogRefreshInterval bounds how often a relay the catalog does+// not list makes the node re-read /info.+const transitCatalogRefreshInterval = 10 * time.Second+// selectTransit decides which of a peer's circuit addresses to dial and// makes sure the node is admitted at the router behind them. A router// relays only between peers it has admitted, and a node keeps sessions@@ -600,30 +618,69 @@ func circuitRelay(ma multiaddr.Multiaddr) peer.ID {// at start, before the circuit is asked for. Circuits through the peer's// other routers are dropped: each would be a handshake and a session// there, and a service the whole mesh calls would hold every caller at-// every one of its routers. A relay that is not a router (a node with-// --enable-relay) is kept and left alone; it admits what it has handshaken-// with on its own terms. Direct addresses always pass.+// every one of its routers.+//+// A relay the catalog does not list is first looked for in a fresh /info:+// the catalog is read at start, a router that was restarting then was not+// in it, and a member that started during a rollout would otherwise never+// transit that router until the next sync. A relay still unlisted after+// that is handshaken with all the same, after the listed routers: a router+// whose lease the control plane does not carry admits the node like any+// other, and a relay that is not a router (a node with --enable-relay)+// admits what it has handshaken with on its own terms and is kept whatever+// the handshake said. Direct addresses always pass.func (n *AgentMeshNode) selectTransit(ctx context.Context, addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr {- routers := map[peer.ID]routerCandidate{}- for _, c := range n.routerCandidates() {- routers[c.ID] = c- }- var viaRouters []peer.ID- seen := map[peer.ID]bool{}+ var relays []peer.ID+ relayAddrs := map[peer.ID][]multiaddr.Multiaddr{}for _, ma := range addrs {relay := circuitRelay(ma)- if _, isRouter := routers[relay]; relay == "" || !isRouter || seen[relay] {+ if relay == "" {continue}- seen[relay] = true- viaRouters = append(viaRouters, relay)+ if _, seen := relayAddrs[relay]; !seen {+ relays = append(relays, relay)+ }+ if via := relayTransportAddr(ma); via != nil {+ relayAddrs[relay] = append(relayAddrs[relay], via)+ } else if _, seen := relayAddrs[relay]; !seen {+ relayAddrs[relay] = nil+ }}- if len(viaRouters) == 0 {+ if len(relays) == 0 {return addrs}+ routers := map[peer.ID]routerCandidate{}+ for _, c := range n.routerCandidates() {+ routers[c.ID] = c+ }+ var unlisted []peer.ID+ for _, r := range relays {+ if _, isRouter := routers[r]; !isRouter {+ unlisted = append(unlisted, r)+ }+ }+ if len(unlisted) > 0 && n.refreshCatalogForTransit(ctx) {+ routers = map[peer.ID]routerCandidate{}+ for _, c := range n.routerCandidates() {+ routers[c.ID] = c+ }+ unlisted = unlisted[:0]+ for _, r := range relays {+ if _, isRouter := routers[r]; !isRouter {+ unlisted = append(unlisted, r)+ }+ }+ }+ var viaRouters []peer.ID+ for _, r := range relays {+ if _, isRouter := routers[r]; isRouter {+ viaRouters = append(viaRouters, r)+ }+ }+usable := map[peer.ID]bool{}- for _, r := range viaRouters {+ for _, r := range relays {if n.isAuthenticatedAndConnected(r) {usable[r] = true}@@ -637,12 +694,20 @@ func (n *AgentMeshNode) selectTransit(ctx context.Context, addrs []multiaddr.Mullogger.Debugf("[AuthN] Cannot authenticate to a transit router: %v", err)return addrs}- for _, r := range viaRouters {- router := peer.AddrInfo{ID: r, Addrs: n.Host.Peerstore().Addrs(r)}+ for _, r := range append(viaRouters, unlisted...) {+ _, listed := routers[r]+ kind := transitListed+ if !listed {+ kind = transitUnlisted+ logger.Infof("[AuthN] Relay %s is not a router the control plane lists; handshaking with it for the circuit", r)+ }+ router := peer.AddrInfo{ID: r, Addrs: append(n.Host.Peerstore().Addrs(r), relayAddrs[r]...)}if err := n.authRouter(ctx, router, biscuitBytes); err != nil {+ transitHandshakesTotal.WithLabelValues(kind, "failed").Inc()logger.Warnf("[AuthN] Transit router %s did not admit this node: %v", r, err)continue}+ transitHandshakesTotal.WithLabelValues(kind, "ok").Inc()logger.Debugf("[AuthN] Authenticated to transit router %s", r)usable[r] = truebreak@@ -662,3 +727,25 @@ func (n *AgentMeshNode) selectTransit(ctx context.Context, addrs []multiaddr.Mul}return kept}++// refreshCatalogForTransit re-reads the routers from the control plane for+// a relay the catalog does not list, at most once per+// transitCatalogRefreshInterval. It reports whether the catalog was re-read.+func (n *AgentMeshNode) refreshCatalogForTransit(ctx context.Context) bool {+ n.mu.Lock()+ due := time.Since(n.transitCatalogRefreshedAt) >= transitCatalogRefreshInterval+ if due {+ n.transitCatalogRefreshedAt = time.Now()+ }+ n.mu.Unlock()+ if !due {+ return false+ }+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)+ defer cancel()+ if err := n.refreshRouterCatalog(ctx); err != nil {+ logger.Debugf("[AuthN] Routers not re-read from the control plane for a transit relay: %v", err)+ return false+ }+ return true+}diff --git a/internal/node/node.go b/internal/node/node.goindex e1334b8a..3caf6228 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -169,23 +169,26 @@ type AgentMeshNode struct {// Options.Routers of them it keeps a session with; shunned are the ones// it stays off until the time given, having been sent away or having// not come back. connMgr protects the attached ones from being trimmed.- routerCatalog []routerCandidate- attached map[peer.ID]bool- shunned map[peer.ID]time.Time- connMgr *connmgr.BasicConnMgr- peerLastEventTime map[string]int64- mu sync.Mutex- nodeConfig *NodeConfigComplete- revokedPeers *lru.Cache[string, int64]- revokedTokens *lru.Cache[string, time.Time]- delegatedBiscuitCache *lru.Cache[string, *api.TokenExchangeResponse]- stsTokenCache *lru.Cache[string, *api.STSTokenResponse]- peerLabelGate *lru.Cache[string, time.Time]- authPeers sync.Map- trustedKeys []TrustedKey- keysMu sync.RWMutex- MeshPolicyRules []biscuit.Rule- MeshPolicyMu sync.RWMutex+ routerCatalog []routerCandidate+ // transitCatalogRefreshedAt is when a relay the catalog did not list+ // last made the node re-read /info (selectTransit).+ transitCatalogRefreshedAt time.Time+ attached map[peer.ID]bool+ shunned map[peer.ID]time.Time+ connMgr *connmgr.BasicConnMgr+ peerLastEventTime map[string]int64+ mu sync.Mutex+ nodeConfig *NodeConfigComplete+ revokedPeers *lru.Cache[string, int64]+ revokedTokens *lru.Cache[string, time.Time]+ delegatedBiscuitCache *lru.Cache[string, *api.TokenExchangeResponse]+ stsTokenCache *lru.Cache[string, *api.STSTokenResponse]+ peerLabelGate *lru.Cache[string, time.Time]+ authPeers sync.Map+ trustedKeys []TrustedKey+ keysMu sync.RWMutex+ MeshPolicyRules []biscuit.Rule+ MeshPolicyMu sync.RWMutex// pendingEgress holds assignments that arrived before Start created the// service registry (SyncControlPlane runs first); Start applies them.pendingEgress []*api.EgressDestination@@ -1153,6 +1156,9 @@ func (n *AgentMeshNode) redialRouter(ctx context.Context, router peer.ID) {}func (n *AgentMeshNode) loadIdentityForAuth() ([]byte, error) {+ if n.Store == nil {+ return nil, fmt.Errorf("%w: no identity store", ErrFatalAuth)+ }biscuitBytes, err := n.Store.LoadIdentity()if err != nil {return nil, fmt.Errorf("%w: failed to load identity from store: %w", ErrFatalAuth, err)diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.goindex 119e7689..aab5824d 100644--- a/internal/node/node_metrics.go+++ b/internal/node/node_metrics.go@@ -41,8 +41,31 @@ func init() {for _, reason := range []api.RouterGoAway_Reason{api.RouterGoAway_DRAINING, api.RouterGoAway_OVERLOADED} {goAwayReceivedTotal.WithLabelValues(reason.String())}+ for _, router := range []string{transitListed, transitUnlisted} {+ for _, result := range []string{"ok", "failed"} {+ transitHandshakesTotal.WithLabelValues(router, result)+ }+ }}+// The router a transit handshake is with: one the control plane's catalog+// lists, or a relay found only in a peer's record.+const (+ transitListed = "listed"+ transitUnlisted = "unlisted"+)++// transitHandshakesTotal counts the handshakes a call made with a router it+// did not hold, to reach a peer through it; the unlisted ones are routers+// the catalog missed, which a rising count during a rollout explains.+var transitHandshakesTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "agentmesh_node_transit_handshakes_total",+ Help: "Handshakes with a router this node did not hold, run to reach a peer through it, by whether the control plane listed the router and by result",+ },+ []string{"router", "result"},+)+// nodeStateCollector exports the node's own view of its mesh membership: the// same figures GET /debug/mesh-info and /debug/token-info answer on demand,// as gauges a scraper can keep. A node that runs unattended, such as adiff --git a/internal/node/router_auth_dedupe_test.go b/internal/node/router_auth_dedupe_test.goindex ee886281..d7aa2676 100644--- a/internal/node/router_auth_dedupe_test.go+++ b/internal/node/router_auth_dedupe_test.go@@ -608,3 +608,44 @@ func TestRouterDropIsRedialed(t *testing.T) {t.Fatalf("handshakes after the second drop = %d, want 3", got)}}++// A peer's record names a circuit through a router the node's catalog does+// not list: the node read the catalog while that router was restarting, or+// the router's lease is not carried right now. The call must handshake with+// that router before asking it for the circuit, as it would with a listed+// one; without a control plane to re-read, the record is all it has.+func TestSelectTransitHandshakesWithARouterTheCatalogDoesNotList(t *testing.T) {+ known := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")+ unlisted := newFakeRouterWithKey(t, known.cpPub, known.cpPriv, "/ip4/127.0.0.1/tcp/0")+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()+ // The node holds the known router; the store carries no control plane+ // URL, so a re-read of the catalog is not possible here.+ node := known.startNode(t, ctx, known.routerAddrs)+ if node.isAuthenticatedAndConnected(unlisted.h.ID()) {+ t.Fatal("node already admitted at the router it was never told about")+ }++ direct := multiaddr.StringCast("/ip4/192.0.2.9/tcp/5002")+ circuit := unlisted.routerAddrs[0].Encapsulate(multiaddr.StringCast("/p2p-circuit"))+ kept := node.selectTransit(ctx, []multiaddr.Multiaddr{direct, circuit})++ if got := unlisted.handshakes.Load(); got != 1 {+ t.Fatalf("unlisted router saw %d handshakes, want 1", got)+ }+ if !node.isAuthenticatedAndConnected(unlisted.h.ID()) {+ t.Fatal("node not admitted at the unlisted router after the handshake")+ }+ var got []string+ for _, a := range kept {+ got = append(got, a.String())+ }+ if len(kept) != 2 || kept[0].String() != direct.String() || kept[1].String() != circuit.String() {+ t.Fatalf("addresses kept: %v, want the direct address and the circuit through the unlisted router", got)+ }+ // Admitted now: a second selection runs no handshake.+ node.selectTransit(ctx, []multiaddr.Multiaddr{circuit})+ if got := unlisted.handshakes.Load(); got != 1 {+ t.Fatalf("unlisted router saw %d handshakes after the node was admitted, want still 1", got)+ }+}diff --git a/internal/node/routers.go b/internal/node/routers.goindex 58236ce5..7f1ba033 100644--- a/internal/node/routers.go+++ b/internal/node/routers.go@@ -173,6 +173,9 @@ func (n *AgentMeshNode) routerCandidates() []routerCandidate {// refreshRouterCatalog re-reads /info. Best effort: a node started from its// stored addresses with no control plane in reach keeps what it has.func (n *AgentMeshNode) refreshRouterCatalog(ctx context.Context) error {+ if n.Store == nil {+ return errors.New("no store")+ }controlPlaneURL, err := n.Store.LoadControlPlaneURL()if err != nil || controlPlaneURL == "" {return errors.New("no control plane URL in store")diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.mdindex fde857a6..3c31ecfe 100644--- a/site/content/docs/guides/scalability.md+++ b/site/content/docs/guides/scalability.md@@ -225,6 +225,7 @@ The router and the control plane serve Prometheus metrics on| `agentmesh_router_goaway_sent_total{reason="OVERLOADED"}` | A router at its high mark sent members to the others. Rising on a router that is not draining, the fleet has outgrown its routers. || `agentmesh_router_inbound_connections_refused_total` | Members behind one address have hit `--conns-per-source-ip`. || `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `scope_limit` more callers at once than the node's connection budget. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. |+| `agentmesh_node_transit_handshakes_total{router,result}` | On a caller: handshakes with routers it did not hold, run to reach a peer through them. `router="unlisted"` is a router the node's catalog missed, which a member that started during a router rollout produces until its next sync; `result="failed"` rising is a router that refuses the node or is gone while peers still announce circuits through it. || `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. || `libp2p_relaysvc_connection_rejections_total{reason="resource limit exceeded"}` | A service has more callers than `--relay-max-circuits`. || `libp2p_rcmgr_blocked_resources` | libp2p refused a connection or stream on a scope. On a router this should stay flat. |diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.goindex 365dc447..59293f5d 100644--- a/tests/integration/router_attachment_test.go+++ b/tests/integration/router_attachment_test.go@@ -232,6 +232,7 @@ func circuitRelaysInRecord(t *testing.T, reader *backgroundNode, target peer.ID)body, _ := io.ReadAll(resp.Body)if resp.StatusCode != http.StatusOK {// Not in the DHT yet.+ t.Logf("peer-addrs for %s: %d %s", target, resp.StatusCode, strings.TrimSpace(string(body)))return nil}var info struct {@@ -718,6 +719,60 @@ func TestCallerTransitsTheRouterItShares(t *testing.T) {}}+// TestCallerTransitsARouterItsCatalogMissed is the member that started+// while a router was away: its catalog, read from /info once at start, does+// not list a router that joined after, and a peer on that router announces+// circuits only through it. A circuit through a router the caller has not+// handshaken with is refused by the relay, so the caller must learn the+// router from the record and admit itself there before it asks.+func TestCallerTransitsARouterItsCatalogMissed(t *testing.T) {+ m := startAttachmentMesh(t)+ routerA := m.startRouter(t, "router-a", "site=a")+ common := append(m.nodeArgs(), "--monitor-bootstrap", "500ms", "--monitor-interval", "500ms")+ // One router wanted and one matching, so the caller is never short and+ // never re-reads /info on its own.+ caller := launchNode(t, m.nodeBin, m.nodeEnv("caller"), filepath.Join(m.tmpDir, "caller"), append(slices.Clone(common),+ "--api-token-path", tokenPath(t, "token-c"),+ "--router-selector", "site=a",+ "--routers", "1",+ )...)+ caller.waitForAPI(t)+ t.Cleanup(func() {+ if t.Failed() {+ t.Logf("--- caller ---\n%s", caller.log())+ }+ })+ waitForEqual(t, "routers attached by the caller", func() []string { return attachedRouters(t, caller) }, peerIDStrings(routerA.peerID), 10*time.Second)++ // Router b arrives after the caller; the provider lives only there.+ routerB := m.startRouter(t, "router-b", "site=b")+ waitForEqual(t, "peer routers router b authenticated", func() []string {+ return []string{fmt.Sprint(routerB.gauge(t, `agentmesh_router_peer_routers{state="authenticated"}`))}+ }, []string{"1"}, 10*time.Second)+ mcpServer := httptest.NewServer(newBoundaryMCPHandler(t))+ t.Cleanup(mcpServer.Close)+ provider := launchNode(t, m.nodeBin, m.nodeEnv("provider"), filepath.Join(m.tmpDir, "provider"), append(slices.Clone(common),+ "--api-token-path", tokenPath(t, "token-p"),+ "--router-selector", "site=b",+ "--config", writeNodeConfig(t, m.tmpDir, nil, svcDecl{Type: "mcp", Name: "late-tool", TargetURL: mcpServer.URL}),+ )...)+ provider.waitForAPI(t)+ waitForEqual(t, "routers attached by the provider", func() []string { return attachedRouters(t, provider) }, peerIDStrings(routerB.peerID), 10*time.Second)+ waitForEqual(t, "routers the provider announces circuits through", func() []string { return announcedCircuitRelays(t, provider) },+ peerIDStrings(routerB.peerID), 15*time.Second)+ // A peer lookup by ID has to connect the peer, and the only path is the+ // circuit through router b, so the record is not readable from here+ // before the handshake there; discovery by service name needs no dial.+ callAcrossMesh(t, caller, "token-c", provider, "late-tool")++ // Router b admitted the provider, router a, the control plane's publisher+ // and, for the call, the caller.+ if got := routerB.gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {+ t.Errorf("router b authenticated %v peers, want 4: the caller must have handshaken there before asking for the circuit", got)+ }+ t.Logf("--- caller ---\n%s", caller.log())+}+// TestRouterKeepsOnlyAdmittedConnections pins what a router holds: a// connection whose peer does not pass the handshake within --auth-deadline// is closed, and the control plane's event publisher, which presents a