Skip to content

Agent Mesh

node: take the address a failed lookup learned, clear the backoff its dial left

Antonio Ojea273e6fbabf25
diff --git a/internal/node/mcp.go b/internal/node/mcp.go
index 4245c684..70d83c52 100644
--- a/internal/node/mcp.go
+++ b/internal/node/mcp.go
@@ -29,6 +29,8 @@ import (
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/core/peerstore"
+ "github.com/libp2p/go-libp2p/core/routing"
+ "github.com/libp2p/go-libp2p/p2p/net/swarm"
"github.com/libp2p/go-msgio"
"github.com/modelcontextprotocol/go-sdk/mcp"
"github.com/multiformats/go-multiaddr"
@@ -469,13 +471,24 @@ func (n *AgentMeshNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID
findCtx, cancel := context.WithTimeout(ctx, dhtLookupTimeout)
addrInfo, err := n.DHT.FindPeer(findCtx, targetPeer)
cancel()
+ addrs = addrInfo.Addrs
if err != nil {
- logger.Debugf("[Discovery] Failed to find peer %s on DHT: %v", targetPeer, err)
- } else {
- addrs = addrInfo.Addrs
- if len(addrs) > 0 {
- changed = true
- }
+ // kad-dht answers "not found" for a peer it could not dial during
+ // the lookup, which a peer reached only through a router this node
+ // has not handshaken with is; what the lookup learned is in the
+ // peerstore all the same.
+ addrs = n.Host.Peerstore().Addrs(targetPeer)
+ }
+ switch {
+ case len(addrs) > 0:
+ changed = true
+ peerLookupsTotal.WithLabelValues(lookupFound).Inc()
+ case err != nil && !errors.Is(err, routing.ErrNotFound):
+ peerLookupsTotal.WithLabelValues(lookupFailed).Inc()
+ logger.Warnf("[Discovery] DHT lookup for %s failed (routing table holds %d peers): %v", targetPeer, n.DHT.RoutingTable().Size(), err)
+ default:
+ peerLookupsTotal.WithLabelValues(lookupNotFound).Inc()
+ logger.Infof("[Discovery] DHT lookup found no address for %s (routing table holds %d peers); trying circuits through this node's routers", targetPeer, n.DHT.RoutingTable().Size())
}
}
logger.Debugf("[Discovery] preparePeerAddrs for %s: found %d addrs in peerstore: %v", targetPeer, len(addrs), addrs)
@@ -573,6 +586,12 @@ func (n *AgentMeshNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID
n.Host.Peerstore().ClearAddrs(targetPeer)
n.Host.Peerstore().AddAddrs(targetPeer, validAddrs, peerstore.TempAddrTTL)
logger.Debugf("[Discovery] Replaced addrs for %s with %d routable/circuit addrs", targetPeer, len(validAddrs))
+ // The lookup dialled the peer before any transit handshake, and a
+ // router refuses a circuit to a caller it has not admitted; the swarm
+ // put that address in dial backoff, which the handshake has outdated.
+ if sw, ok := n.Host.Network().(*swarm.Swarm); ok {
+ sw.Backoff().Clear(targetPeer)
+ }
}
}
diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.go
index aab5824d..1b2dbaaf 100644
--- a/internal/node/node_metrics.go
+++ b/internal/node/node_metrics.go
@@ -46,8 +46,30 @@ func init() {
transitHandshakesTotal.WithLabelValues(router, result)
}
}
+ for _, result := range []string{lookupFound, lookupNotFound, lookupFailed} {
+ peerLookupsTotal.WithLabelValues(result)
+ }
}
+// The outcome of a DHT lookup for a peer's addresses: found, found
+// nothing, or the lookup itself failed (an empty routing table, a timeout).
+const (
+ lookupFound = "found"
+ lookupNotFound = "not_found"
+ lookupFailed = "failed"
+)
+
+// peerLookupsTotal counts the DHT lookups a call ran for a peer it held no
+// address for; not_found rising is a peer the DHT does not carry, failed
+// rising is this node cut off from the DHT.
+var peerLookupsTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_node_peer_lookups_total",
+ Help: "DHT lookups for a peer's addresses, by outcome: found, not_found, failed",
+ },
+ []string{"result"},
+)
+
// The router a transit handshake is with: one the control plane's catalog
// lists, or a relay found only in a peer's record.
const (
diff --git a/internal/node/peer_lookup_test.go b/internal/node/peer_lookup_test.go
new file mode 100644
index 00000000..59c7b360
--- /dev/null
+++ b/internal/node/peer_lookup_test.go
@@ -0,0 +1,91 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/google/agentmesh/api"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ "github.com/libp2p/go-libp2p/core/test"
+ "github.com/libp2p/go-libp2p/p2p/net/swarm"
+ "github.com/multiformats/go-multiaddr"
+)
+
+// A peer reached only through a router this node has not handshaken with:
+// the lookup learns the circuit address from a router that holds it, dials
+// it to query the peer, and is refused, since that router has not admitted
+// this node; kad-dht then answers "not found" and leaves the address in
+// dial backoff. The node must take the address the lookup learned anyway,
+// handshake with the router, and clear the backoff, or the call fails
+// twice over for a peer that is one handshake away.
+func TestPreparePeerAddrsUsesWhatAFailedLookupLearned(t *testing.T) {
+ held := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ // The held router is a DHT server, as a router is, so it enters the
+ // node's routing table and answers its lookups.
+ heldDHT, err := dht.New(held.h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix(api.DHTProtocolPrefix))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = heldDHT.Close() }()
+ other := newFakeRouterWithKey(t, held.cpPub, held.cpPriv, "/ip4/127.0.0.1/tcp/0")
+
+ // The peer reserved with the other router, which recorded the circuit;
+ // the held router learned it, as a router does through identify.
+ target := test.RandPeerIDFatal(t)
+ circuit := other.routerAddrs[0].Encapsulate(multiaddr.StringCast("/p2p-circuit"))
+ held.h.Peerstore().AddAddrs(target, []multiaddr.Multiaddr{circuit}, time.Hour)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ node := held.startNode(t, ctx, held.routerAddrs)
+ for node.DHT.RoutingTable().Size() == 0 {
+ select {
+ case <-ctx.Done():
+ t.Fatal("the held router never entered the node's routing table")
+ case <-time.After(20 * time.Millisecond):
+ }
+ }
+ before := counterValue(t, peerLookupsTotal.WithLabelValues(lookupFound))
+
+ node.preparePeerAddrs(ctx, target)
+
+ if got := other.handshakes.Load(); got != 1 {
+ t.Fatalf("the other router saw %d handshakes, want 1: the node must handshake with the router the lookup named", got)
+ }
+ var got []string
+ for _, a := range node.Host.Peerstore().Addrs(target) {
+ got = append(got, a.String())
+ }
+ slices.Sort(got)
+ want := []string{circuit.String(), "/p2p/" + other.h.ID().String() + "/p2p-circuit"}
+ slices.Sort(want)
+ if !slices.Equal(got, want) {
+ t.Fatalf("addresses kept for the peer: %v, want the circuit the lookup learned and its bare form %v", got, want)
+ }
+ if after := counterValue(t, peerLookupsTotal.WithLabelValues(lookupFound)); after != before+1 {
+ t.Errorf("lookups counted as found: %v, want %v", after, before+1)
+ }
+ sw, ok := node.Host.Network().(*swarm.Swarm)
+ if !ok {
+ t.Fatal("the node's network is not a swarm")
+ }
+ if sw.Backoff().Backoff(target, circuit) {
+ t.Error("the circuit the lookup's refused dial put in backoff is still there after the handshake")
+ }
+}
diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.md
index 3135c7c6..760b2366 100644
--- a/site/content/docs/guides/scalability.md
+++ b/site/content/docs/guides/scalability.md
@@ -236,6 +236,7 @@ The router and the control plane serve Prometheus metrics on
| `agentmesh_router_inbound_connections_refused_total` | Members behind one address have hit `--conns-per-source-ip`. |
| `agentmesh_node_inbound_connections_refused_total{reason}` against `agentmesh_node_conns_per_source_ip_limit` | On a provider: callers it turned away before the handshake. `per_ip_limit` is a fleet behind one address, `scope_limit` more callers at once than the node's connection budget. The caller sees the connection close before TLS, or `CONNECTION_REFUSED` over QUIC, and nothing more. |
| `agentmesh_node_transit_handshakes_total{router,result}` | On a caller: handshakes with routers it did not hold, run to reach a peer through them. `router="unlisted"` is a router the node's catalog missed, which a member that started during a router rollout produces until its next sync; `result="failed"` rising is a router that refuses the node or is gone while peers still announce circuits through it. |
+| `agentmesh_node_peer_lookups_total{result}` | On a caller: DHT lookups for a peer it held no address for. `not_found` is a peer the DHT does not carry, and the call falls back to circuits through the caller's own routers, which reach the peer only if it holds one of them; `failed` rising is the caller cut off from the DHT (`agentmesh_node_dht_routing_table_size` at 0). |
| `libp2p_relaysvc_reservations_total{type="opened"}` minus `{type="closed"}`, against `agentmesh_router_relay_limit{limit="reservations"}` | Reservations in use against the budget. |
| `libp2p_relaysvc_connection_rejections_total{reason="resource limit exceeded"}` | A service has more callers than `--relay-max-circuits`. |
| `libp2p_rcmgr_blocked_resources` | libp2p refused a connection or stream on a scope. On a router this should stay flat. |