Skip to content

Agent Mesh

router: serve the DHT and gossip to authenticated peers only

Antonio Ojea1aaca97ef048
diff --git a/internal/router/auth_gate.go b/internal/router/auth_gate.go
new file mode 100644
index 00000000..22e1d06e
--- /dev/null
+++ b/internal/router/auth_gate.go
@@ -0,0 +1,128 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/protocol"
+)
+
+// streamScopeUnauthenticated labels a stream the gate refused: its peer's
+// connection ended before the peer authenticated.
+const streamScopeUnauthenticated = "unauthenticated"
+
+// authPollInterval is how often a stream waiting on its peer's handshake
+// checks for it.
+const authPollInterval = 100 * time.Millisecond
+
+// defaultAuthWait bounds the wait for a peer's handshake when the router
+// runs without an auth deadline, which would otherwise close the connection.
+const defaultAuthWait = time.Minute
+
+var errNotAuthenticated = errors.New("peer did not authenticate")
+
+// authGate is the host the DHT and gossipsub are built on. A library
+// registers its stream handlers through it, and a handler so registered
+// serves a peer only once that peer completed the mesh handshake on this
+// router; a stream the library opens waits for the same. The handshake
+// itself, identify and the relay keep the plain host: the first two are
+// what the handshake needs, the relay checks admission in its own ACL.
+//
+// A stream that arrives before its peer's handshake is held, not reset: a
+// member's DHT and gossip streams open on connect, a moment before its
+// handshake completes, and a reset would make each library drop the peer
+// and retry on its own schedule. An unauthenticated peer's stream is held
+// unserved until its connection closes at the auth deadline, and counted.
+type authGate struct {
+ host.Host
+ r *Router
+}
+
+func (g *authGate) SetStreamHandler(pid protocol.ID, h network.StreamHandler) {
+ g.Host.SetStreamHandler(pid, g.r.requireAuthenticated(h))
+}
+
+func (g *authGate) SetStreamHandlerMatch(pid protocol.ID, match func(protocol.ID) bool, h network.StreamHandler) {
+ g.Host.SetStreamHandlerMatch(pid, match, g.r.requireAuthenticated(h))
+}
+
+func (g *authGate) NewStream(ctx context.Context, p peer.ID, pids ...protocol.ID) (network.Stream, error) {
+ if err := g.r.awaitAuthenticated(ctx, p); err != nil {
+ return nil, err
+ }
+ return g.Host.NewStream(ctx, p, pids...)
+}
+
+// requireAuthenticated wraps a stream handler so it runs only once the
+// stream's peer has authenticated.
+func (r *Router) requireAuthenticated(next network.StreamHandler) network.StreamHandler {
+ return func(s network.Stream) {
+ p := s.Conn().RemotePeer()
+ if err := r.awaitAuthenticated(r.lifetime(), p); err != nil {
+ streamsRefusedTotal.WithLabelValues(streamScopeUnauthenticated, string(s.Protocol())).Inc()
+ logger.Debugf("[AuthZ] Refusing %s stream from %s: %v", s.Protocol(), p, err)
+ _ = s.Reset()
+ return
+ }
+ next(s)
+ }
+}
+
+// awaitAuthenticated returns once p has authenticated, or with an error
+// when ctx ends, p's connections are gone, or the auth deadline passes.
+func (r *Router) awaitAuthenticated(ctx context.Context, p peer.ID) error {
+ if r.isPeerAuthenticated(p) {
+ return nil
+ }
+ wait := r.config.AuthDeadline
+ if wait <= 0 {
+ wait = defaultAuthWait
+ }
+ parent := ctx
+ ctx, cancel := context.WithTimeout(ctx, wait)
+ defer cancel()
+ tick := time.NewTicker(authPollInterval)
+ defer tick.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ if parent.Err() != nil {
+ return parent.Err()
+ }
+ return errNotAuthenticated
+ case <-tick.C:
+ }
+ if r.isPeerAuthenticated(p) {
+ return nil
+ }
+ if r.Host.Network().Connectedness(p) != network.Connected {
+ return errNotAuthenticated
+ }
+ }
+}
+
+// lifetime is the router's own context, for work not tied to a request.
+func (r *Router) lifetime() context.Context {
+ if r.ctx != nil {
+ return r.ctx
+ }
+ return context.Background()
+}
diff --git a/internal/router/auth_gate_test.go b/internal/router/auth_gate_test.go
new file mode 100644
index 00000000..990d8045
--- /dev/null
+++ b/internal/router/auth_gate_test.go
@@ -0,0 +1,247 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "context"
+ "errors"
+ "io"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ "github.com/libp2p/go-libp2p/core/event"
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/protocol"
+ dto "github.com/prometheus/client_model/go"
+
+ "github.com/google/agentmesh/api"
+)
+
+const echoProtocol protocol.ID = "/test/echo/1.0.0"
+
+// newGatedRouter is a router whose DHT and gossip host is the gate, with a
+// short auth deadline so an unauthenticated peer is cut off quickly.
+func newGatedRouter(t *testing.T) (*Router, *authGate) {
+ t.Helper()
+ h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = h.Close() })
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+ r := &Router{Host: h, ctx: ctx, directory: api.DefaultDirectorySettings(), config: Options{AuthDeadline: 2 * time.Second}}
+ return r, &authGate{Host: h, r: r}
+}
+
+func newClient(t *testing.T, r *Router) host.Host {
+ t.Helper()
+ c, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = c.Close() })
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := c.Connect(ctx, peer.AddrInfo{ID: r.Host.ID(), Addrs: r.Host.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+ return c
+}
+
+// waitForInboundStream returns once the router holds a negotiated stream
+// on proto from p, so the gate's handler has been dispatched for it.
+func waitForInboundStream(t *testing.T, r *Router, p peer.ID, proto protocol.ID) {
+ t.Helper()
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ for _, conn := range r.Host.Network().ConnsToPeer(p) {
+ for _, st := range conn.GetStreams() {
+ if st.Protocol() == proto {
+ return
+ }
+ }
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("the router never saw a %s stream from %s", proto, p)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+}
+
+func unauthenticatedRefusals(t *testing.T, proto protocol.ID) float64 {
+ t.Helper()
+ var m dto.Metric
+ if err := streamsRefusedTotal.WithLabelValues(streamScopeUnauthenticated, string(proto)).Write(&m); err != nil {
+ t.Fatal(err)
+ }
+ return m.GetCounter().GetValue()
+}
+
+// A handler registered through the gate serves a peer only once it
+// authenticated: a stream that arrives first waits for the handshake and is
+// then served; a peer that never authenticates gets nothing, and its stream
+// is counted as refused when its connection closes at the auth deadline.
+func TestGateServesOnlyAuthenticatedPeers(t *testing.T) {
+ r, gated := newGatedRouter(t)
+ gated.SetStreamHandler(echoProtocol, func(s network.Stream) {
+ defer func() { _ = s.Close() }()
+ _, _ = io.Copy(s, s)
+ })
+
+ t.Run("a stream opened before the handshake is served after it", func(t *testing.T) {
+ c := newClient(t, r)
+ s, err := c.NewStream(context.Background(), r.Host.ID(), echoProtocol)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = s.Close() }()
+ if _, err := s.Write([]byte("hello")); err != nil {
+ t.Fatal(err)
+ }
+ buf := make([]byte, 5)
+ _ = s.SetReadDeadline(time.Now().Add(300 * time.Millisecond))
+ if n, err := s.Read(buf); err == nil {
+ t.Fatalf("read %q before the handshake; the stream must wait for it", buf[:n])
+ }
+
+ r.authenticatedPeers.Store(c.ID(), time.Now().Add(time.Hour))
+ _ = s.SetReadDeadline(time.Now().Add(5 * time.Second))
+ if _, err := io.ReadFull(s, buf); err != nil {
+ t.Fatalf("read after the handshake: %v", err)
+ }
+ if string(buf) != "hello" {
+ t.Fatalf("got %q, want %q", buf, "hello")
+ }
+ })
+
+ t.Run("a peer that never authenticates is refused and counted", func(t *testing.T) {
+ before := unauthenticatedRefusals(t, echoProtocol)
+ c := newClient(t, r)
+ s, err := c.NewStream(context.Background(), r.Host.ID(), echoProtocol)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := s.Write([]byte("hello")); err != nil {
+ t.Fatal(err)
+ }
+ waitForInboundStream(t, r, c.ID(), echoProtocol)
+ // The connection goes, as it would at the auth deadline.
+ if err := c.Network().ClosePeer(r.Host.ID()); err != nil {
+ t.Fatal(err)
+ }
+ deadline := time.Now().Add(5 * time.Second)
+ for unauthenticatedRefusals(t, echoProtocol) != before+1 {
+ if time.Now().After(deadline) {
+ t.Fatalf("refusals = %v, want %v: a stream from a peer that never authenticated must be counted", unauthenticatedRefusals(t, echoProtocol), before+1)
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ })
+
+ t.Run("a stream the router opens waits for the handshake", func(t *testing.T) {
+ c := newClient(t, r)
+ c.SetStreamHandler(echoProtocol, func(s network.Stream) { _ = s.Close() })
+ ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+ defer cancel()
+ if _, err := gated.NewStream(ctx, c.ID(), echoProtocol); !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("NewStream to an unauthenticated peer: err = %v, want the context's deadline", err)
+ }
+ r.authenticatedPeers.Store(c.ID(), time.Now().Add(time.Hour))
+ s, err := gated.NewStream(context.Background(), c.ID(), echoProtocol)
+ if err != nil {
+ t.Fatalf("NewStream to an authenticated peer: %v", err)
+ }
+ _ = s.Close()
+ })
+}
+
+// The router's DHT, built on the gate, answers a member's query only once
+// the member authenticated, and its table admits authenticated peer routers
+// and nothing else.
+func TestRouterDHTServesAuthenticatedMembersAndTablesPeerRouters(t *testing.T) {
+ r, gated := newGatedRouter(t)
+ kdht, err := dht.New(gated, r.dhtOptions()...)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = kdht.Close() })
+ r.DHT = kdht
+
+ // A member: a DHT client that asks the router for the peers closest to a key.
+ member := newClient(t, r)
+ memberDHT, err := dht.New(member, dht.Mode(dht.ModeClient), dht.ProtocolPrefix(api.DHTProtocolPrefix), dht.BucketSize(3), dht.DisableAutoRefresh())
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = memberDHT.Close() })
+ if _, err := memberDHT.RoutingTable().TryAddPeer(r.Host.ID(), true, false); err != nil {
+ t.Fatal(err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+ if _, err := memberDHT.GetClosestPeers(ctx, "a key"); err == nil {
+ t.Fatal("an unauthenticated member's query was answered")
+ }
+ cancel()
+
+ r.authenticatedPeers.Store(member.ID(), time.Now().Add(time.Hour))
+ ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if _, err := memberDHT.GetClosestPeers(ctx, "a key"); err != nil {
+ t.Fatalf("an authenticated member's query failed: %v", err)
+ }
+ if kdht.RoutingTable().Find(member.ID()) != "" {
+ t.Fatal("a member entered the router's routing table")
+ }
+
+ // A peer router in server mode: in the table once it authenticates, not
+ // before. The DHT hears of a peer when identify completes.
+ identified, err := r.Host.EventBus().Subscribe(new(event.EvtPeerIdentificationCompleted))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = identified.Close() })
+ other := newClient(t, r)
+ otherDHT, err := dht.New(other, dht.Mode(dht.ModeServer), dht.ProtocolPrefix(api.DHTProtocolPrefix))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = otherDHT.Close() })
+ for seen := false; !seen; {
+ select {
+ case e := <-identified.Out():
+ seen = e.(event.EvtPeerIdentificationCompleted).Peer == other.ID()
+ case <-time.After(5 * time.Second):
+ t.Fatal("the router never identified the peer")
+ }
+ }
+ if kdht.RoutingTable().Find(other.ID()) != "" {
+ t.Fatal("a server-mode peer that never authenticated entered the routing table")
+ }
+ r.authenticatedPeers.Store(other.ID(), time.Now().Add(time.Hour))
+ r.admitPeerRouter(other.ID())
+ if kdht.RoutingTable().Find(other.ID()) == "" {
+ t.Fatal("an authenticated peer router is not in the routing table")
+ }
+ r.forgetPeer(other.ID())
+ if kdht.RoutingTable().Find(other.ID()) != "" {
+ t.Fatal("a peer router whose authentication lapsed is still in the routing table")
+ }
+}
diff --git a/internal/router/dht.go b/internal/router/dht.go
index 083ef6c8..44a5a4a5 100644
--- a/internal/router/dht.go
+++ b/internal/router/dht.go
@@ -20,6 +20,7 @@ import (
dht "github.com/libp2p/go-libp2p-kad-dht"
"github.com/libp2p/go-libp2p-kad-dht/records"
+ "github.com/libp2p/go-libp2p/core/peer"
"github.com/google/agentmesh/api"
)
@@ -40,12 +41,46 @@ const dhtTableBucketSize = 64
// node that is gone drops out of discovery within it and two missed
// announcements do not unlist one that is not. The library default (48h)
// would keep every past identity listed.
-func dhtOptions(directory api.DirectorySettings) []dht.Option {
+//
+// Its table holds, and its lookups ask, authenticated peer routers only: a
+// peer in server mode that never authenticated would otherwise be handed
+// queries and records. A router enters the table when it authenticates
+// (admitPeerRouter), not when the DHT first hears of it.
+func (r *Router) dhtOptions() []dht.Option {
return []dht.Option{
dht.Mode(dht.ModeServer),
dht.ProtocolPrefix(api.DHTProtocolPrefix),
dht.BucketSize(dhtTableBucketSize),
- dht.ProviderManagerOpts(records.ProviderAddrTTL(directory.RecordTTL), records.ProvideValidity(directory.RecordTTL)),
+ dht.ProviderManagerOpts(records.ProviderAddrTTL(r.directory.RecordTTL), records.ProvideValidity(r.directory.RecordTTL)),
+ dht.RoutingTableFilter(func(_ interface{}, p peer.ID) bool { return r.isAuthenticatedPeerRouter(p) }),
+ dht.QueryFilter(func(_ interface{}, ai peer.AddrInfo) bool { return r.isAuthenticatedPeerRouter(ai.ID) }),
+ }
+}
+
+// isAuthenticatedPeerRouter reports whether p is a peer router that holds a
+// current authentication with this router.
+func (r *Router) isAuthenticatedPeerRouter(p peer.ID) bool {
+ if _, isRouter := r.peerRouters.Load(p); !isRouter {
+ return false
+ }
+ return r.isPeerAuthenticated(p)
+}
+
+// admitPeerRouter records p as a peer router that authenticated and puts it
+// in the DHT's table: the table's filter kept it out until now, and the
+// DHT itself only adds a peer when it first hears of it.
+func (r *Router) admitPeerRouter(p peer.ID) {
+ if _, known := r.peerRouters.LoadOrStore(p, true); !known {
+ logger.Infof("[Federation] Peer router %s authenticated with us", p)
+ r.Host.ConnManager().Protect(p, federationTag)
+ }
+ if r.DHT == nil {
+ return
+ }
+ if added, err := r.DHT.RoutingTable().TryAddPeer(p, true, false); err != nil {
+ logger.Warnf("[DHT] Peer router %s not added to the routing table: %v", p, err)
+ } else if added {
+ logger.Infof("[DHT] Peer router %s added to the routing table (%d routers)", p, r.DHT.RoutingTable().Size())
}
}
diff --git a/internal/router/dht_test.go b/internal/router/dht_test.go
index 4f51cbd1..9025539c 100644
--- a/internal/router/dht_test.go
+++ b/internal/router/dht_test.go
@@ -32,7 +32,8 @@ func TestRouterDHTTableHoldsEveryRouter(t *testing.T) {
t.Fatalf("libp2p.New: %v", err)
}
t.Cleanup(func() { _ = h.Close() })
- kdht, err := dht.New(h, dhtOptions(api.DefaultDirectorySettings())...)
+ r := &Router{Host: h, directory: api.DefaultDirectorySettings()}
+ kdht, err := dht.New(h, r.dhtOptions()...)
if err != nil {
t.Fatalf("dht.New: %v", err)
}
@@ -43,13 +44,19 @@ func TestRouterDHTTableHoldsEveryRouter(t *testing.T) {
rt := kdht.RoutingTable()
const routers = 40
- for i := 0; i < routers; i++ {
+ for added := 0; added < routers; {
p, err := rt.GenRandPeerID(0)
if err != nil {
t.Fatalf("GenRandPeerID: %v", err)
}
+ // GenRandPeerID draws from a 16-bit prefix table, so it repeats a
+ // peer now and then; a repeat is not a refusal.
+ if rt.Find(p) != "" {
+ continue
+ }
if ok, err := rt.TryAddPeer(p, true, false); !ok {
- t.Fatalf("router #%d refused from the table: %v; a bucket must hold at least %d", i+1, err, routers)
+ t.Fatalf("router #%d refused from the table: %v; a bucket must hold at least %d", added+1, err, routers)
}
+ added++
}
}
diff --git a/internal/router/router.go b/internal/router/router.go
index 6b778146..42face05 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -252,6 +252,9 @@ func (r *Router) forgetPeer(p peer.ID) {
r.Host.ConnManager().UntagPeer(p, memberTag)
r.Host.Peerstore().UpdateAddrs(p, r.reservationTTL(), 0)
}
+ if r.DHT != nil {
+ r.DHT.RoutingTable().RemovePeer(p)
+ }
}
// AdmittedUntil reports when this router's admission of p lapses: the
@@ -502,9 +505,11 @@ func (r *Router) Start() (retErr error) {
return err
}
r.Host = hostNode
+ // The DHT and gossip serve, and are served by, authenticated peers only.
+ gated := &authGate{Host: hostNode, r: r}
// Setup DHT
- kadDHT, err := dht.New(hostNode, dhtOptions(r.directory)...)
+ kadDHT, err := dht.New(gated, r.dhtOptions()...)
if err != nil {
return err
}
@@ -526,7 +531,7 @@ func (r *Router) Start() (retErr error) {
// Setup PubSub. StrictSign is the default; pinned because the event
// validator trusts msg.GetFrom().
- ps, err := pubsub.NewGossipSub(r.ctx, hostNode, pubsub.WithMessageSignaturePolicy(pubsub.StrictSign))
+ ps, err := pubsub.NewGossipSub(r.ctx, gated, pubsub.WithMessageSignaturePolicy(pubsub.StrictSign))
if err != nil {
return err
}
@@ -1509,10 +1514,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
// A router that proves the router role is a peer router from here on,
// not from the next federation pass: it is never a member to shed.
if identity.RequireRole(b, verifyingKey, api.RoleRouter, r.config.BiscuitTimeout) == nil {
- if _, known := r.peerRouters.LoadOrStore(remotePeer, true); !known {
- logger.Infof("[Federation] Peer router %s authenticated with us", remotePeer)
- r.Host.ConnManager().Protect(remotePeer, federationTag)
- }
+ r.admitPeerRouter(remotePeer)
}
// Send mutual response (our biscuit)
@@ -1589,6 +1591,7 @@ func (r *Router) performMutualAuth(s network.Stream) error {
}
r.admitPeer(remotePeer, expiry)
+ r.admitPeerRouter(remotePeer)
return nil
}