Skip to content

Agent Mesh

node: handshake with every router it connects to, and keep the session

Antonio Ojea6dd99e1724fb
diff --git a/internal/node/debug_handlers.go b/internal/node/debug_handlers.go
index 5924c457..f9b05cac 100644
--- a/internal/node/debug_handlers.go
+++ b/internal/node/debug_handlers.go
@@ -132,9 +132,12 @@ type meshInfoResponse struct {
PeerID string `json:"peer_id"`
ConnectedPeers []string `json:"connected_peers"`
AttachedRouters []string `json:"attached_routers"`
- DHTSize int `json:"dht_size"`
- RouterPeerID string `json:"router_peer_id"`
- LocalAPISocket string `json:"local_api_socket,omitempty"`
+ // DirectoryRouters are the routers the node holds a session with for
+ // its DHT alone: the ones its keys landed on, not attached.
+ DirectoryRouters []string `json:"directory_routers"`
+ DHTSize int `json:"dht_size"`
+ RouterPeerID string `json:"router_peer_id"`
+ LocalAPISocket string `json:"local_api_socket,omitempty"`
}
type connectivityResponse struct {
@@ -191,14 +194,21 @@ func (n *AgentMeshNode) meshInfo() (*meshInfoResponse, error) {
attachedRouters = append(attachedRouters, p.String())
}
slices.Sort(attachedRouters)
+ directory := n.directoryRouters()
+ directoryRouters := make([]string, 0, len(directory))
+ for _, p := range directory {
+ directoryRouters = append(directoryRouters, p.String())
+ }
+ slices.Sort(directoryRouters)
return &meshInfoResponse{
- PeerID: n.Host.ID().String(),
- ConnectedPeers: connectedPeers,
- AttachedRouters: attachedRouters,
- DHTSize: n.DHT.RoutingTable().Size(),
- RouterPeerID: n.RouterPeerID.String(),
- LocalAPISocket: n.BoundSocketPath,
+ PeerID: n.Host.ID().String(),
+ ConnectedPeers: connectedPeers,
+ AttachedRouters: attachedRouters,
+ DirectoryRouters: directoryRouters,
+ DHTSize: n.DHT.RoutingTable().Size(),
+ RouterPeerID: n.RouterPeerID.String(),
+ LocalAPISocket: n.BoundSocketPath,
}, nil
}
diff --git a/internal/node/directory.go b/internal/node/directory.go
new file mode 100644
index 00000000..ec44351d
--- /dev/null
+++ b/internal/node/directory.go
@@ -0,0 +1,85 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+// authDirectoryRouter runs the credential handshake with a router this node
+// just connected to without being attached to it. The DHT client opens such
+// connections for a lookup or an announcement, to the routers kademlia puts
+// closest to the key; left anonymous, the router closes them at its auth
+// deadline a minute later and the next lookup opens them again. Handshaken,
+// they are sessions: the router keeps them, counts them, and the node's
+// lookups reuse them. The routers a node holds this way are its K attached
+// ones plus those its keys land on, which is how a mesh with more routers
+// spreads them; a node's own connection manager trims the idle ones first.
+// Attached routers handshake on attach, and a connection that arrives while
+// that handshake is in flight waits for it (authRouter).
+func (n *AgentMeshNode) authDirectoryRouter(ctx context.Context, router peer.ID) {
+ if n.isAttached(router) || n.isAuthenticatedAndConnected(router) || !n.isCatalogRouter(router) {
+ return
+ }
+ go func() {
+ biscuitBytes, err := n.loadIdentityForAuth()
+ if err != nil {
+ logger.Debugf("[AuthN] Cannot authenticate to directory router %s without an identity: %v", router, err)
+ return
+ }
+ if err := n.authRouter(ctx, peer.AddrInfo{ID: router}, biscuitBytes); err != nil {
+ directoryHandshakesTotal.WithLabelValues("failed").Inc()
+ logger.Warnf("[AuthN] Directory router %s did not admit this node; the router will close the connection at its auth deadline: %v", router, err)
+ return
+ }
+ directoryHandshakesTotal.WithLabelValues("ok").Inc()
+ logger.Infof("[AuthN] Authenticated to directory router %s", router)
+ }()
+}
+
+// isCatalogRouter reports whether p is a router the control plane lists.
+func (n *AgentMeshNode) isCatalogRouter(p peer.ID) bool {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ for _, c := range n.routerCatalog {
+ if c.ID == p {
+ return true
+ }
+ }
+ return false
+}
+
+// directoryRouters are the routers this node holds an authenticated session
+// with for its DHT alone: connected and admitted, not attached.
+func (n *AgentMeshNode) directoryRouters() []peer.ID {
+ attached := map[peer.ID]bool{}
+ n.mu.Lock()
+ for r := range n.attached {
+ attached[r] = true
+ }
+ n.mu.Unlock()
+ var out []peer.ID
+ for _, r := range n.authenticatedRouterIDs() {
+ if !attached[r] {
+ out = append(out, r)
+ }
+ }
+ return out
+}
+
+// DirectoryRouters is how many routers this node holds for its DHT alone.
+func (n *AgentMeshNode) DirectoryRouters() int { return len(n.directoryRouters()) }
diff --git a/internal/node/directory_test.go b/internal/node/directory_test.go
new file mode 100644
index 00000000..01dc190f
--- /dev/null
+++ b/internal/node/directory_test.go
@@ -0,0 +1,72 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+// TestConnectingToAnUnattachedRouterHandshakes: a connection the node opens
+// to a catalog router it is not attached to, as its DHT does for a lookup,
+// is handshaken once and counted as a directory session; the attached
+// router's handshake is not repeated by it.
+func TestConnectingToAnUnattachedRouterHandshakes(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+
+ r1 := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ r2 := newFakeRouterWithKey(t, r1.cpPub, r1.cpPriv, "/ip4/127.0.0.1/tcp/0")
+ node := r1.startNode(t, ctx, append(r1.routerAddrs, r2.routerAddrs...), func(o *Options) { o.Routers = 1 })
+
+ attached := node.attachedRouters()
+ if len(attached) != 1 {
+ t.Fatalf("attached routers = %v, want one", attached)
+ }
+ other := r2
+ if attached[0] == r2.h.ID() {
+ other = r1
+ }
+ if got := node.DirectoryRouters(); got != 0 {
+ t.Fatalf("directory routers before any lookup = %d, want 0", got)
+ }
+
+ // What the DHT client does when a key lands on the other router.
+ if err := node.Host.Connect(ctx, peer.AddrInfo{ID: other.h.ID(), Addrs: other.h.Addrs()}); err != nil {
+ t.Fatalf("Connect: %v", err)
+ }
+ deadline := time.Now().Add(5 * time.Second)
+ for !node.isAuthenticatedAndConnected(other.h.ID()) {
+ if time.Now().After(deadline) {
+ t.Fatalf("node did not authenticate to %s after connecting to it; handshakes seen there: %d", other.h.ID(), other.handshakes.Load())
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ if got := other.handshakes.Load(); got != 1 {
+ t.Errorf("unattached router saw %d handshakes, want 1", got)
+ }
+ if got := node.DirectoryRouters(); got != 1 {
+ t.Errorf("directory routers = %d, want 1", got)
+ }
+ if got := node.AttachedRouters(); got != 1 {
+ t.Errorf("attached routers after the directory handshake = %d, want 1", got)
+ }
+ if r1.handshakes.Load()+r2.handshakes.Load() != 2 {
+ t.Errorf("handshakes in all = %d, want 2: one per router", r1.handshakes.Load()+r2.handshakes.Load())
+ }
+}
diff --git a/internal/node/node.go b/internal/node/node.go
index 060bd1e5..f8c3c651 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -157,6 +157,9 @@ type AgentMeshNode struct {
authenticatedRouters map[peer.ID]bool
// redialing holds the routers a redialRouter goroutine is working on.
redialing map[peer.ID]bool
+ // handshaking holds the routers a handshake is in flight with; a second
+ // caller waits for that one (authRouter).
+ handshaking map[peer.ID]chan struct{}
// routerCatalog is every router the node knows of; attached are the
// Options.Routers of them it keeps a session with; shunned are the ones
// it stays off until the time given, having been sent away or having
@@ -636,7 +639,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
n.Host = h
h.Network().Notify(&network.NotifyBundle{
- ConnectedF: func(n network.Network, c network.Conn) {
+ ConnectedF: func(_ network.Network, c network.Conn) {
remotePeer := c.RemotePeer()
remoteAddr := c.RemoteMultiaddr()
if hasCircuit(remoteAddr) || !isPrivateIP(remoteAddr) {
@@ -645,6 +648,9 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
}
logger.Debugf("[Discovery] Peer %s connected via relay/public IP (%s), marking private IP as failed", remotePeer, remoteAddr)
}
+ if c.Stat().Direction == network.DirOutbound {
+ n.authDirectoryRouter(ctx, remotePeer)
+ }
},
// The router drops us from its authenticated set the moment our last
// connection to it closes, so a stale entry here would report a
@@ -1184,12 +1190,41 @@ func (n *AgentMeshNode) loadIdentityForAuth() ([]byte, error) {
// and runs the handshake, unless a live authenticated session to it already
// exists: Host.Connect is a no-op on an existing connection, but the
// handshake is not, and repeating it trips the router's per-peer handshake
-// limiter. The dial is bounded by RouterConnectTimeout so a router that is
-// down does not hold its caller for long.
+// limiter. For the same reason two callers reaching for the same router at
+// once, as the attach path and the directory path do when its connection
+// comes up, run one handshake: the second waits for the first. The dial is
+// bounded by RouterConnectTimeout so a router that is down does not hold
+// its caller for long.
func (n *AgentMeshNode) authRouter(ctx context.Context, router peer.AddrInfo, biscuitBytes []byte) error {
if n.isAuthenticatedAndConnected(router.ID) {
return nil
}
+ n.mu.Lock()
+ if done, inFlight := n.handshaking[router.ID]; inFlight {
+ n.mu.Unlock()
+ select {
+ case <-done:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ if n.isAuthenticatedAndConnected(router.ID) {
+ return nil
+ }
+ return fmt.Errorf("router %s: the handshake in flight did not admit this node", router.ID)
+ }
+ if n.handshaking == nil {
+ n.handshaking = map[peer.ID]chan struct{}{}
+ }
+ done := make(chan struct{})
+ n.handshaking[router.ID] = done
+ n.mu.Unlock()
+ defer func() {
+ n.mu.Lock()
+ delete(n.handshaking, router.ID)
+ n.mu.Unlock()
+ close(done)
+ }()
+
routerCtx, cancel := context.WithTimeout(ctx, n.config.RouterConnectTimeout)
defer cancel()
if err := n.Host.Connect(routerCtx, router); err != nil {
diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.go
index ebd65547..f8da526f 100644
--- a/internal/node/node_metrics.go
+++ b/internal/node/node_metrics.go
@@ -101,6 +101,17 @@ var transitHandshakesTotal = promauto.NewCounterVec(
[]string{"router", "result"},
)
+// directoryHandshakesTotal counts the handshakes run with a router this node
+// connected to for its DHT without being attached to it, so the router
+// keeps the connection as a session; see authDirectoryRouter.
+var directoryHandshakesTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_node_directory_handshakes_total",
+ Help: "Handshakes with a router this node connected to for a DHT lookup or announcement without being attached to it, by result",
+ },
+ []string{"result"},
+)
+
// nodeStateCollector exports the node's own view of its mesh membership: the
// same figures GET /debug/mesh-info and /debug/token-info answer on demand,
// as gauges a scraper can keep. A node that runs unattended, such as a
@@ -152,7 +163,7 @@ func newNodeStateCollector(n *AgentMeshNode) *nodeStateCollector {
[]string{"type"}, nil),
routersDesc: prometheus.NewDesc(
"agentmesh_node_routers",
- "Routers this node keeps a session with: attached now, and wanted (--routers, or every router listed when fewer)",
+ "Routers this node keeps a session with: attached now, wanted (--routers, or every router listed when fewer), and directory (authenticated for DHT lookups and announcements without being attached)",
[]string{"state"}, nil),
routerCandsDesc: prometheus.NewDesc(
"agentmesh_node_router_candidates",
@@ -188,6 +199,7 @@ func (c *nodeStateCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.AttachedRouters()), "attached")
ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.wantRouters()), "wanted")
+ ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.DirectoryRouters()), "directory")
cands := n.routerCandidates()
_, matching := selectRouters(cands, 0, n.config.RouterSelector, nil, nil)
ch <- prometheus.MustNewConstMetric(c.routerCandsDesc, prometheus.GaugeValue, float64(len(cands)), "any")
diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.go
index 59293f5d..fc3cc180 100644
--- a/tests/integration/router_attachment_test.go
+++ b/tests/integration/router_attachment_test.go
@@ -26,6 +26,7 @@ import (
"os/exec"
"path/filepath"
"slices"
+ "strconv"
"strings"
"syscall"
"testing"
@@ -151,17 +152,43 @@ func routersConnectedTo(t *testing.T, cpPort int, peerID peer.ID) []string {
// meshInfo reads the node's own view: the routers it holds and every peer
// it has a connection with.
func meshInfo(t *testing.T, n *backgroundNode) (attached, connected []string) {
+ t.Helper()
+ attached, _, connected = meshView(t, n)
+ return attached, connected
+}
+
+// meshView is meshInfo with the directory routers as well: the routers the
+// node holds a session with for its DHT alone.
+func meshView(t *testing.T, n *backgroundNode) (attached, directory, connected []string) {
t.Helper()
var info struct {
- AttachedRouters []string `json:"attached_routers"`
- ConnectedPeers []string `json:"connected_peers"`
+ AttachedRouters []string `json:"attached_routers"`
+ DirectoryRouters []string `json:"directory_routers"`
+ ConnectedPeers []string `json:"connected_peers"`
}
if err := json.Unmarshal([]byte(debugGetWithToken(t, n.apiAddr, n.token, "/debug/mesh-info")), &info); err != nil {
t.Fatalf("decode mesh-info: %v", err)
}
slices.Sort(info.AttachedRouters)
+ slices.Sort(info.DirectoryRouters)
slices.Sort(info.ConnectedPeers)
- return info.AttachedRouters, info.ConnectedPeers
+ return info.AttachedRouters, info.DirectoryRouters, info.ConnectedPeers
+}
+
+// nodeCounter reads one counter series from the node's /metrics, 0 when it
+// has not been emitted.
+func nodeCounter(t *testing.T, n *backgroundNode, series string) float64 {
+ t.Helper()
+ for _, line := range strings.Split(debugGetWithToken(t, n.apiAddr, n.token, "/metrics"), "\n") {
+ if strings.HasPrefix(line, series+" ") {
+ v, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimPrefix(line, series)), 64)
+ if err != nil {
+ t.Fatalf("parse %s: %v", line, err)
+ }
+ return v
+ }
+ }
+ return 0
}
// attachedRouters reads the routers the node holds as relays from its own
@@ -585,11 +612,25 @@ func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {
}
}
}
+ // A router's sessions are its own node, the two peer routers, the
+ // control plane's event publisher, and the nodes whose keys landed on
+ // it: the directory routers each node reports, which are routers and
+ // nothing else.
for _, s := range sites {
- // Its own node, the two peer routers and the control plane's
- // event publisher.
- if got := routers[s].gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {
- t.Errorf("router %s authenticated %v peers, want 4 (its node, two routers, the control plane)", s, got)
+ want := 4.0
+ for _, other := range sites {
+ _, directory, _ := meshView(t, nodes[other])
+ for _, r := range directory {
+ if !slices.Contains(peerIDStrings(routers["a"].peerID, routers["b"].peerID, routers["c"].peerID), r) {
+ t.Errorf("node %s holds a directory session with %s, which is not a router", other, r)
+ }
+ if r == routers[s].peerID.String() {
+ want++
+ }
+ }
+ }
+ if got := routers[s].gauge(t, "agentmesh_router_authenticated_peers"); got != want {
+ t.Errorf("router %s authenticated %v peers, want %v (its node, two routers, the control plane, and the nodes that hold it for their keys)", s, got, want)
}
}
if t.Failed() {
@@ -706,13 +747,20 @@ func TestCallerTransitsTheRouterItShares(t *testing.T) {
callAcrossMesh(t, caller, "token-c", provider, "shared-tool")
- // Router a authenticated the provider, the two routers and the control
- // plane's publisher; the caller never came. Router b has both nodes.
- if got := routerA.gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {
- t.Errorf("router a authenticated %v peers, want 4: the caller must transit router b, which it holds", got)
+ // The caller holds router b, which the provider's record names, so the
+ // call went through it and no handshake was run for transit. Router a
+ // may still hold the caller for its DHT: the provider's key lands on
+ // the closest routers, which on a mesh of three is all of them.
+ for _, kind := range []string{"listed", "unlisted"} {
+ for _, result := range []string{"ok", "failed"} {
+ series := fmt.Sprintf(`agentmesh_node_transit_handshakes_total{result="%s",router="%s"}`, result, kind)
+ if got := nodeCounter(t, caller, series); got != 0 {
+ t.Errorf("%s = %v, want 0: the caller must transit router b, which it holds", series, got)
+ }
+ }
}
- if got := routerB.gauge(t, "agentmesh_router_authenticated_peers"); got != 5 {
- t.Errorf("router b authenticated %v peers, want 5", got)
+ if got := routerB.gauge(t, "agentmesh_router_authenticated_peers"); got < 5 {
+ t.Errorf("router b authenticated %v peers, want at least 5: both nodes, two routers, the control plane", got)
}
if t.Failed() {
t.Logf("--- caller ---\n%s", caller.log())