Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: serve the DHT and gossip to authenticated peers only
Antonio Ojea1aaca97ef048
diff --git a/internal/router/auth_gate.go b/internal/router/auth_gate.gonew file mode 100644index 00000000..22e1d06e--- /dev/null+++ b/internal/router/auth_gate.go@@ -0,0 +1,128 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package router++import (+ "context"+ "errors"+ "time"++ "github.com/libp2p/go-libp2p/core/host"+ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-libp2p/core/protocol"+)++// streamScopeUnauthenticated labels a stream the gate refused: its peer's+// connection ended before the peer authenticated.+const streamScopeUnauthenticated = "unauthenticated"++// authPollInterval is how often a stream waiting on its peer's handshake+// checks for it.+const authPollInterval = 100 * time.Millisecond++// defaultAuthWait bounds the wait for a peer's handshake when the router+// runs without an auth deadline, which would otherwise close the connection.+const defaultAuthWait = time.Minute++var errNotAuthenticated = errors.New("peer did not authenticate")++// authGate is the host the DHT and gossipsub are built on. A library+// registers its stream handlers through it, and a handler so registered+// serves a peer only once that peer completed the mesh handshake on this+// router; a stream the library opens waits for the same. The handshake+// itself, identify and the relay keep the plain host: the first two are+// what the handshake needs, the relay checks admission in its own ACL.+//+// A stream that arrives before its peer's handshake is held, not reset: a+// member's DHT and gossip streams open on connect, a moment before its+// handshake completes, and a reset would make each library drop the peer+// and retry on its own schedule. An unauthenticated peer's stream is held+// unserved until its connection closes at the auth deadline, and counted.+type authGate struct {+ host.Host+ r *Router+}++func (g *authGate) SetStreamHandler(pid protocol.ID, h network.StreamHandler) {+ g.Host.SetStreamHandler(pid, g.r.requireAuthenticated(h))+}++func (g *authGate) SetStreamHandlerMatch(pid protocol.ID, match func(protocol.ID) bool, h network.StreamHandler) {+ g.Host.SetStreamHandlerMatch(pid, match, g.r.requireAuthenticated(h))+}++func (g *authGate) NewStream(ctx context.Context, p peer.ID, pids ...protocol.ID) (network.Stream, error) {+ if err := g.r.awaitAuthenticated(ctx, p); err != nil {+ return nil, err+ }+ return g.Host.NewStream(ctx, p, pids...)+}++// requireAuthenticated wraps a stream handler so it runs only once the+// stream's peer has authenticated.+func (r *Router) requireAuthenticated(next network.StreamHandler) network.StreamHandler {+ return func(s network.Stream) {+ p := s.Conn().RemotePeer()+ if err := r.awaitAuthenticated(r.lifetime(), p); err != nil {+ streamsRefusedTotal.WithLabelValues(streamScopeUnauthenticated, string(s.Protocol())).Inc()+ logger.Debugf("[AuthZ] Refusing %s stream from %s: %v", s.Protocol(), p, err)+ _ = s.Reset()+ return+ }+ next(s)+ }+}++// awaitAuthenticated returns once p has authenticated, or with an error+// when ctx ends, p's connections are gone, or the auth deadline passes.+func (r *Router) awaitAuthenticated(ctx context.Context, p peer.ID) error {+ if r.isPeerAuthenticated(p) {+ return nil+ }+ wait := r.config.AuthDeadline+ if wait <= 0 {+ wait = defaultAuthWait+ }+ parent := ctx+ ctx, cancel := context.WithTimeout(ctx, wait)+ defer cancel()+ tick := time.NewTicker(authPollInterval)+ defer tick.Stop()+ for {+ select {+ case <-ctx.Done():+ if parent.Err() != nil {+ return parent.Err()+ }+ return errNotAuthenticated+ case <-tick.C:+ }+ if r.isPeerAuthenticated(p) {+ return nil+ }+ if r.Host.Network().Connectedness(p) != network.Connected {+ return errNotAuthenticated+ }+ }+}++// lifetime is the router's own context, for work not tied to a request.+func (r *Router) lifetime() context.Context {+ if r.ctx != nil {+ return r.ctx+ }+ return context.Background()+}diff --git a/internal/router/auth_gate_test.go b/internal/router/auth_gate_test.gonew file mode 100644index 00000000..990d8045--- /dev/null+++ b/internal/router/auth_gate_test.go@@ -0,0 +1,247 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package router++import (+ "context"+ "errors"+ "io"+ "testing"+ "time"++ "github.com/libp2p/go-libp2p"+ dht "github.com/libp2p/go-libp2p-kad-dht"+ "github.com/libp2p/go-libp2p/core/event"+ "github.com/libp2p/go-libp2p/core/host"+ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-libp2p/core/protocol"+ dto "github.com/prometheus/client_model/go"++ "github.com/google/agentmesh/api"+)++const echoProtocol protocol.ID = "/test/echo/1.0.0"++// newGatedRouter is a router whose DHT and gossip host is the gate, with a+// short auth deadline so an unauthenticated peer is cut off quickly.+func newGatedRouter(t *testing.T) (*Router, *authGate) {+ t.Helper()+ h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = h.Close() })+ ctx, cancel := context.WithCancel(context.Background())+ t.Cleanup(cancel)+ r := &Router{Host: h, ctx: ctx, directory: api.DefaultDirectorySettings(), config: Options{AuthDeadline: 2 * time.Second}}+ return r, &authGate{Host: h, r: r}+}++func newClient(t *testing.T, r *Router) host.Host {+ t.Helper()+ c, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = c.Close() })+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ if err := c.Connect(ctx, peer.AddrInfo{ID: r.Host.ID(), Addrs: r.Host.Addrs()}); err != nil {+ t.Fatal(err)+ }+ return c+}++// waitForInboundStream returns once the router holds a negotiated stream+// on proto from p, so the gate's handler has been dispatched for it.+func waitForInboundStream(t *testing.T, r *Router, p peer.ID, proto protocol.ID) {+ t.Helper()+ deadline := time.Now().Add(5 * time.Second)+ for {+ for _, conn := range r.Host.Network().ConnsToPeer(p) {+ for _, st := range conn.GetStreams() {+ if st.Protocol() == proto {+ return+ }+ }+ }+ if time.Now().After(deadline) {+ t.Fatalf("the router never saw a %s stream from %s", proto, p)+ }+ time.Sleep(10 * time.Millisecond)+ }+}++func unauthenticatedRefusals(t *testing.T, proto protocol.ID) float64 {+ t.Helper()+ var m dto.Metric+ if err := streamsRefusedTotal.WithLabelValues(streamScopeUnauthenticated, string(proto)).Write(&m); err != nil {+ t.Fatal(err)+ }+ return m.GetCounter().GetValue()+}++// A handler registered through the gate serves a peer only once it+// authenticated: a stream that arrives first waits for the handshake and is+// then served; a peer that never authenticates gets nothing, and its stream+// is counted as refused when its connection closes at the auth deadline.+func TestGateServesOnlyAuthenticatedPeers(t *testing.T) {+ r, gated := newGatedRouter(t)+ gated.SetStreamHandler(echoProtocol, func(s network.Stream) {+ defer func() { _ = s.Close() }()+ _, _ = io.Copy(s, s)+ })++ t.Run("a stream opened before the handshake is served after it", func(t *testing.T) {+ c := newClient(t, r)+ s, err := c.NewStream(context.Background(), r.Host.ID(), echoProtocol)+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = s.Close() }()+ if _, err := s.Write([]byte("hello")); err != nil {+ t.Fatal(err)+ }+ buf := make([]byte, 5)+ _ = s.SetReadDeadline(time.Now().Add(300 * time.Millisecond))+ if n, err := s.Read(buf); err == nil {+ t.Fatalf("read %q before the handshake; the stream must wait for it", buf[:n])+ }++ r.authenticatedPeers.Store(c.ID(), time.Now().Add(time.Hour))+ _ = s.SetReadDeadline(time.Now().Add(5 * time.Second))+ if _, err := io.ReadFull(s, buf); err != nil {+ t.Fatalf("read after the handshake: %v", err)+ }+ if string(buf) != "hello" {+ t.Fatalf("got %q, want %q", buf, "hello")+ }+ })++ t.Run("a peer that never authenticates is refused and counted", func(t *testing.T) {+ before := unauthenticatedRefusals(t, echoProtocol)+ c := newClient(t, r)+ s, err := c.NewStream(context.Background(), r.Host.ID(), echoProtocol)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := s.Write([]byte("hello")); err != nil {+ t.Fatal(err)+ }+ waitForInboundStream(t, r, c.ID(), echoProtocol)+ // The connection goes, as it would at the auth deadline.+ if err := c.Network().ClosePeer(r.Host.ID()); err != nil {+ t.Fatal(err)+ }+ deadline := time.Now().Add(5 * time.Second)+ for unauthenticatedRefusals(t, echoProtocol) != before+1 {+ if time.Now().After(deadline) {+ t.Fatalf("refusals = %v, want %v: a stream from a peer that never authenticated must be counted", unauthenticatedRefusals(t, echoProtocol), before+1)+ }+ time.Sleep(20 * time.Millisecond)+ }+ })++ t.Run("a stream the router opens waits for the handshake", func(t *testing.T) {+ c := newClient(t, r)+ c.SetStreamHandler(echoProtocol, func(s network.Stream) { _ = s.Close() })+ ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)+ defer cancel()+ if _, err := gated.NewStream(ctx, c.ID(), echoProtocol); !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("NewStream to an unauthenticated peer: err = %v, want the context's deadline", err)+ }+ r.authenticatedPeers.Store(c.ID(), time.Now().Add(time.Hour))+ s, err := gated.NewStream(context.Background(), c.ID(), echoProtocol)+ if err != nil {+ t.Fatalf("NewStream to an authenticated peer: %v", err)+ }+ _ = s.Close()+ })+}++// The router's DHT, built on the gate, answers a member's query only once+// the member authenticated, and its table admits authenticated peer routers+// and nothing else.+func TestRouterDHTServesAuthenticatedMembersAndTablesPeerRouters(t *testing.T) {+ r, gated := newGatedRouter(t)+ kdht, err := dht.New(gated, r.dhtOptions()...)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = kdht.Close() })+ r.DHT = kdht++ // A member: a DHT client that asks the router for the peers closest to a key.+ member := newClient(t, r)+ memberDHT, err := dht.New(member, dht.Mode(dht.ModeClient), dht.ProtocolPrefix(api.DHTProtocolPrefix), dht.BucketSize(3), dht.DisableAutoRefresh())+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = memberDHT.Close() })+ if _, err := memberDHT.RoutingTable().TryAddPeer(r.Host.ID(), true, false); err != nil {+ t.Fatal(err)+ }++ ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)+ if _, err := memberDHT.GetClosestPeers(ctx, "a key"); err == nil {+ t.Fatal("an unauthenticated member's query was answered")+ }+ cancel()++ r.authenticatedPeers.Store(member.ID(), time.Now().Add(time.Hour))+ ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ if _, err := memberDHT.GetClosestPeers(ctx, "a key"); err != nil {+ t.Fatalf("an authenticated member's query failed: %v", err)+ }+ if kdht.RoutingTable().Find(member.ID()) != "" {+ t.Fatal("a member entered the router's routing table")+ }++ // A peer router in server mode: in the table once it authenticates, not+ // before. The DHT hears of a peer when identify completes.+ identified, err := r.Host.EventBus().Subscribe(new(event.EvtPeerIdentificationCompleted))+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = identified.Close() })+ other := newClient(t, r)+ otherDHT, err := dht.New(other, dht.Mode(dht.ModeServer), dht.ProtocolPrefix(api.DHTProtocolPrefix))+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = otherDHT.Close() })+ for seen := false; !seen; {+ select {+ case e := <-identified.Out():+ seen = e.(event.EvtPeerIdentificationCompleted).Peer == other.ID()+ case <-time.After(5 * time.Second):+ t.Fatal("the router never identified the peer")+ }+ }+ if kdht.RoutingTable().Find(other.ID()) != "" {+ t.Fatal("a server-mode peer that never authenticated entered the routing table")+ }+ r.authenticatedPeers.Store(other.ID(), time.Now().Add(time.Hour))+ r.admitPeerRouter(other.ID())+ if kdht.RoutingTable().Find(other.ID()) == "" {+ t.Fatal("an authenticated peer router is not in the routing table")+ }+ r.forgetPeer(other.ID())+ if kdht.RoutingTable().Find(other.ID()) != "" {+ t.Fatal("a peer router whose authentication lapsed is still in the routing table")+ }+}diff --git a/internal/router/dht.go b/internal/router/dht.goindex 083ef6c8..44a5a4a5 100644--- a/internal/router/dht.go+++ b/internal/router/dht.go@@ -20,6 +20,7 @@ import (dht "github.com/libp2p/go-libp2p-kad-dht""github.com/libp2p/go-libp2p-kad-dht/records"+ "github.com/libp2p/go-libp2p/core/peer""github.com/google/agentmesh/api")@@ -40,12 +41,46 @@ const dhtTableBucketSize = 64// node that is gone drops out of discovery within it and two missed// announcements do not unlist one that is not. The library default (48h)// would keep every past identity listed.-func dhtOptions(directory api.DirectorySettings) []dht.Option {+//+// Its table holds, and its lookups ask, authenticated peer routers only: a+// peer in server mode that never authenticated would otherwise be handed+// queries and records. A router enters the table when it authenticates+// (admitPeerRouter), not when the DHT first hears of it.+func (r *Router) dhtOptions() []dht.Option {return []dht.Option{dht.Mode(dht.ModeServer),dht.ProtocolPrefix(api.DHTProtocolPrefix),dht.BucketSize(dhtTableBucketSize),- dht.ProviderManagerOpts(records.ProviderAddrTTL(directory.RecordTTL), records.ProvideValidity(directory.RecordTTL)),+ dht.ProviderManagerOpts(records.ProviderAddrTTL(r.directory.RecordTTL), records.ProvideValidity(r.directory.RecordTTL)),+ dht.RoutingTableFilter(func(_ interface{}, p peer.ID) bool { return r.isAuthenticatedPeerRouter(p) }),+ dht.QueryFilter(func(_ interface{}, ai peer.AddrInfo) bool { return r.isAuthenticatedPeerRouter(ai.ID) }),+ }+}++// isAuthenticatedPeerRouter reports whether p is a peer router that holds a+// current authentication with this router.+func (r *Router) isAuthenticatedPeerRouter(p peer.ID) bool {+ if _, isRouter := r.peerRouters.Load(p); !isRouter {+ return false+ }+ return r.isPeerAuthenticated(p)+}++// admitPeerRouter records p as a peer router that authenticated and puts it+// in the DHT's table: the table's filter kept it out until now, and the+// DHT itself only adds a peer when it first hears of it.+func (r *Router) admitPeerRouter(p peer.ID) {+ if _, known := r.peerRouters.LoadOrStore(p, true); !known {+ logger.Infof("[Federation] Peer router %s authenticated with us", p)+ r.Host.ConnManager().Protect(p, federationTag)+ }+ if r.DHT == nil {+ return+ }+ if added, err := r.DHT.RoutingTable().TryAddPeer(p, true, false); err != nil {+ logger.Warnf("[DHT] Peer router %s not added to the routing table: %v", p, err)+ } else if added {+ logger.Infof("[DHT] Peer router %s added to the routing table (%d routers)", p, r.DHT.RoutingTable().Size())}}diff --git a/internal/router/dht_test.go b/internal/router/dht_test.goindex 4f51cbd1..9025539c 100644--- a/internal/router/dht_test.go+++ b/internal/router/dht_test.go@@ -32,7 +32,8 @@ func TestRouterDHTTableHoldsEveryRouter(t *testing.T) {t.Fatalf("libp2p.New: %v", err)}t.Cleanup(func() { _ = h.Close() })- kdht, err := dht.New(h, dhtOptions(api.DefaultDirectorySettings())...)+ r := &Router{Host: h, directory: api.DefaultDirectorySettings()}+ kdht, err := dht.New(h, r.dhtOptions()...)if err != nil {t.Fatalf("dht.New: %v", err)}@@ -43,13 +44,19 @@ func TestRouterDHTTableHoldsEveryRouter(t *testing.T) {rt := kdht.RoutingTable()const routers = 40- for i := 0; i < routers; i++ {+ for added := 0; added < routers; {p, err := rt.GenRandPeerID(0)if err != nil {t.Fatalf("GenRandPeerID: %v", err)}+ // GenRandPeerID draws from a 16-bit prefix table, so it repeats a+ // peer now and then; a repeat is not a refusal.+ if rt.Find(p) != "" {+ continue+ }if ok, err := rt.TryAddPeer(p, true, false); !ok {- t.Fatalf("router #%d refused from the table: %v; a bucket must hold at least %d", i+1, err, routers)+ t.Fatalf("router #%d refused from the table: %v; a bucket must hold at least %d", added+1, err, routers)}+ added++}}diff --git a/internal/router/router.go b/internal/router/router.goindex 6b778146..42face05 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -252,6 +252,9 @@ func (r *Router) forgetPeer(p peer.ID) {r.Host.ConnManager().UntagPeer(p, memberTag)r.Host.Peerstore().UpdateAddrs(p, r.reservationTTL(), 0)}+ if r.DHT != nil {+ r.DHT.RoutingTable().RemovePeer(p)+ }}// AdmittedUntil reports when this router's admission of p lapses: the@@ -502,9 +505,11 @@ func (r *Router) Start() (retErr error) {return err}r.Host = hostNode+ // The DHT and gossip serve, and are served by, authenticated peers only.+ gated := &authGate{Host: hostNode, r: r}// Setup DHT- kadDHT, err := dht.New(hostNode, dhtOptions(r.directory)...)+ kadDHT, err := dht.New(gated, r.dhtOptions()...)if err != nil {return err}@@ -526,7 +531,7 @@ func (r *Router) Start() (retErr error) {// Setup PubSub. StrictSign is the default; pinned because the event// validator trusts msg.GetFrom().- ps, err := pubsub.NewGossipSub(r.ctx, hostNode, pubsub.WithMessageSignaturePolicy(pubsub.StrictSign))+ ps, err := pubsub.NewGossipSub(r.ctx, gated, pubsub.WithMessageSignaturePolicy(pubsub.StrictSign))if err != nil {return err}@@ -1509,10 +1514,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {// A router that proves the router role is a peer router from here on,// not from the next federation pass: it is never a member to shed.if identity.RequireRole(b, verifyingKey, api.RoleRouter, r.config.BiscuitTimeout) == nil {- if _, known := r.peerRouters.LoadOrStore(remotePeer, true); !known {- logger.Infof("[Federation] Peer router %s authenticated with us", remotePeer)- r.Host.ConnManager().Protect(remotePeer, federationTag)- }+ r.admitPeerRouter(remotePeer)}// Send mutual response (our biscuit)@@ -1589,6 +1591,7 @@ func (r *Router) performMutualAuth(s network.Stream) error {}r.admitPeer(remotePeer, expiry)+ r.admitPeerRouter(remotePeer)return nil}