Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: take the address a failed lookup learned, clear the backoff its dial left
Antonio Ojea273e6fbabf25
diff --git a/internal/node/mcp.go b/internal/node/mcp.goindex 4245c684..70d83c52 100644--- a/internal/node/mcp.go+++ b/internal/node/mcp.go@@ -29,6 +29,8 @@ import ("github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer""github.com/libp2p/go-libp2p/core/peerstore"+ "github.com/libp2p/go-libp2p/core/routing"+ "github.com/libp2p/go-libp2p/p2p/net/swarm""github.com/libp2p/go-msgio""github.com/modelcontextprotocol/go-sdk/mcp""github.com/multiformats/go-multiaddr"@@ -469,13 +471,24 @@ func (n *AgentMeshNode) preparePeerAddrs(ctx context.Context, targetPeer peer.IDfindCtx, cancel := context.WithTimeout(ctx, dhtLookupTimeout)addrInfo, err := n.DHT.FindPeer(findCtx, targetPeer)cancel()+ addrs = addrInfo.Addrsif err != nil {- logger.Debugf("[Discovery] Failed to find peer %s on DHT: %v", targetPeer, err)- } else {- addrs = addrInfo.Addrs- if len(addrs) > 0 {- changed = true- }+ // kad-dht answers "not found" for a peer it could not dial during+ // the lookup, which a peer reached only through a router this node+ // has not handshaken with is; what the lookup learned is in the+ // peerstore all the same.+ addrs = n.Host.Peerstore().Addrs(targetPeer)+ }+ switch {+ case len(addrs) > 0:+ changed = true+ peerLookupsTotal.WithLabelValues(lookupFound).Inc()+ case err != nil && !errors.Is(err, routing.ErrNotFound):+ peerLookupsTotal.WithLabelValues(lookupFailed).Inc()+ logger.Warnf("[Discovery] DHT lookup for %s failed (routing table holds %d peers): %v", targetPeer, n.DHT.RoutingTable().Size(), err)+ default:+ peerLookupsTotal.WithLabelValues(lookupNotFound).Inc()+ logger.Infof("[Discovery] DHT lookup found no address for %s (routing table holds %d peers); trying circuits through this node's routers", targetPeer, n.DHT.RoutingTable().Size())}}logger.Debugf("[Discovery] preparePeerAddrs for %s: found %d addrs in peerstore: %v", targetPeer, len(addrs), addrs)@@ -573,6 +586,12 @@ func (n *AgentMeshNode) preparePeerAddrs(ctx context.Context, targetPeer peer.IDn.Host.Peerstore().ClearAddrs(targetPeer)n.Host.Peerstore().AddAddrs(targetPeer, validAddrs, peerstore.TempAddrTTL)logger.Debugf("[Discovery] Replaced addrs for %s with %d routable/circuit addrs", targetPeer, len(validAddrs))+ // The lookup dialled the peer before any transit handshake, and a+ // router refuses a circuit to a caller it has not admitted; the swarm+ // put that address in dial backoff, which the handshake has outdated.+ if sw, ok := n.Host.Network().(*swarm.Swarm); ok {+ sw.Backoff().Clear(targetPeer)+ }}}diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.goindex aab5824d..1b2dbaaf 100644--- a/internal/node/node_metrics.go+++ b/internal/node/node_metrics.go@@ -46,8 +46,30 @@ func init() {transitHandshakesTotal.WithLabelValues(router, result)}}+ for _, result := range []string{lookupFound, lookupNotFound, lookupFailed} {+ peerLookupsTotal.WithLabelValues(result)+ }}+// The outcome of a DHT lookup for a peer's addresses: found, found+// nothing, or the lookup itself failed (an empty routing table, a timeout).+const (+ lookupFound = "found"+ lookupNotFound = "not_found"+ lookupFailed = "failed"+)++// peerLookupsTotal counts the DHT lookups a call ran for a peer it held no+// address for; not_found rising is a peer the DHT does not carry, failed+// rising is this node cut off from the DHT.+var peerLookupsTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "agentmesh_node_peer_lookups_total",+ Help: "DHT lookups for a peer's addresses, by outcome: found, not_found, failed",+ },+ []string{"result"},+)+// The router a transit handshake is with: one the control plane's catalog// lists, or a relay found only in a peer's record.const (diff --git a/internal/node/peer_lookup_test.go b/internal/node/peer_lookup_test.gonew file mode 100644index 00000000..59c7b360--- /dev/null+++ b/internal/node/peer_lookup_test.go@@ -0,0 +1,91 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "context"+ "slices"+ "testing"+ "time"++ "github.com/google/agentmesh/api"+ dht "github.com/libp2p/go-libp2p-kad-dht"+ "github.com/libp2p/go-libp2p/core/test"+ "github.com/libp2p/go-libp2p/p2p/net/swarm"+ "github.com/multiformats/go-multiaddr"+)++// A peer reached only through a router this node has not handshaken with:+// the lookup learns the circuit address from a router that holds it, dials+// it to query the peer, and is refused, since that router has not admitted+// this node; kad-dht then answers "not found" and leaves the address in+// dial backoff. The node must take the address the lookup learned anyway,+// handshake with the router, and clear the backoff, or the call fails+// twice over for a peer that is one handshake away.+func TestPreparePeerAddrsUsesWhatAFailedLookupLearned(t *testing.T) {+ held := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")+ // The held router is a DHT server, as a router is, so it enters the+ // node's routing table and answers its lookups.+ heldDHT, err := dht.New(held.h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix(api.DHTProtocolPrefix))+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = heldDHT.Close() }()+ other := newFakeRouterWithKey(t, held.cpPub, held.cpPriv, "/ip4/127.0.0.1/tcp/0")++ // The peer reserved with the other router, which recorded the circuit;+ // the held router learned it, as a router does through identify.+ target := test.RandPeerIDFatal(t)+ circuit := other.routerAddrs[0].Encapsulate(multiaddr.StringCast("/p2p-circuit"))+ held.h.Peerstore().AddAddrs(target, []multiaddr.Multiaddr{circuit}, time.Hour)++ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()+ node := held.startNode(t, ctx, held.routerAddrs)+ for node.DHT.RoutingTable().Size() == 0 {+ select {+ case <-ctx.Done():+ t.Fatal("the held router never entered the node's routing table")+ case <-time.After(20 * time.Millisecond):+ }+ }+ before := counterValue(t, peerLookupsTotal.WithLabelValues(lookupFound))++ node.preparePeerAddrs(ctx, target)++ if got := other.handshakes.Load(); got != 1 {+ t.Fatalf("the other router saw %d handshakes, want 1: the node must handshake with the router the lookup named", got)+ }+ var got []string+ for _, a := range node.Host.Peerstore().Addrs(target) {+ got = append(got, a.String())+ }+ slices.Sort(got)+ want := []string{circuit.String(), "/p2p/" + other.h.ID().String() + "/p2p-circuit"}+ slices.Sort(want)+ if !slices.Equal(got, want) {+ t.Fatalf("addresses kept for the peer: %v, want the circuit the lookup learned and its bare form %v", got, want)+ }+ if after := counterValue(t, peerLookupsTotal.WithLabelValues(lookupFound)); after != before+1 {+ t.Errorf("lookups counted as found: %v, want %v", after, before+1)+ }+ sw, ok := node.Host.Network().(*swarm.Swarm)+ if !ok {+ t.Fatal("the node's network is not a swarm")+ }+ if sw.Backoff().Backoff(target, circuit) {+ t.Error("the circuit the lookup's refused dial put in backoff is still there after the handshake")+ }+}diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.mdindex 3135c7c6..760b2366 100644--- a/site/content/docs/guides/scalability.md+++ b/site/content/docs/guides/scalability.md@@ -236,6 +236,7 @@ The router and the control plane serve Prometheus metrics on| `agentmesh_router_inbound_connections_refused_total` | Members behind one address have hit `--conns-per-source-ip`. || `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `scope_limit` more callers at once than the node's connection budget. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. || `agentmesh_node_transit_handshakes_total{router,result}` | On a caller: handshakes with routers it did not hold, run to reach a peer through them. `router="unlisted"` is a router the node's catalog missed, which a member that started during a router rollout produces until its next sync; `result="failed"` rising is a router that refuses the node or is gone while peers still announce circuits through it. |+| `agentmesh_node_peer_lookups_total{result}` | On a caller: DHT lookups for a peer it held no address for. `not_found` is a peer the DHT does not carry, and the call falls back to circuits through the caller's own routers, which reach the peer only if it holds one of them; `failed` rising is the caller cut off from the DHT (`agentmesh_node_dht_routing_table_size` at 0). || `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. || `libp2p_relaysvc_connection_rejections_total{reason="resource limit exceeded"}` | A service has more callers than `--relay-max-circuits`. || `libp2p_rcmgr_blocked_resources` | libp2p refused a connection or stream on a scope. On a router this should stay flat. |