Skip to content

Agent Mesh

router: serve the directory to members over /mesh/directory

Antonio Ojeaaf95c1d8eb34
diff --git a/go.mod b/go.mod
index 45d084a0..a087e4ff 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ require (
github.com/libp2p/go-libp2p-gostream v0.6.0
github.com/libp2p/go-libp2p-http v0.5.0
github.com/libp2p/go-libp2p-kad-dht v0.42.2
+ github.com/libp2p/go-libp2p-kbucket v0.9.0
github.com/libp2p/go-libp2p-pubsub v0.18.0
github.com/libp2p/go-msgio v0.3.0
github.com/mattn/go-isatty v0.0.24
@@ -80,7 +81,6 @@ require (
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.3.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
- github.com/libp2p/go-libp2p-kbucket v0.9.0 // indirect
github.com/libp2p/go-libp2p-record v0.3.1 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.5 // indirect
github.com/libp2p/go-netroute v0.4.0 // indirect
diff --git a/internal/router/directory.go b/internal/router/directory.go
new file mode 100644
index 00000000..517ccc24
--- /dev/null
+++ b/internal/router/directory.go
@@ -0,0 +1,402 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "sync"
+ "time"
+
+ "github.com/ipfs/go-cid"
+ kb "github.com/libp2p/go-libp2p-kbucket"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-msgio"
+ "github.com/multiformats/go-multiaddr"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+ "google.golang.org/protobuf/proto"
+
+ "github.com/google/agentmesh/api"
+)
+
+const (
+ // directoryRequestTimeout bounds one request, the lookup across the
+ // routers included. A lookup on a mesh of routers that all know each
+ // other is one round; the bound is for a router that stopped answering.
+ directoryRequestTimeout = 20 * time.Second
+ // directoryMaxMessage bounds a request or response frame.
+ directoryMaxMessage = 64 * 1024
+ // directoryDefaultLimit and directoryMaxLimit bound how many providers
+ // a find_providers returns.
+ directoryDefaultLimit = 20
+ directoryMaxLimit = 100
+)
+
+const (
+ directoryOpProvide = "provide"
+ directoryOpFindProviders = "find_providers"
+ directoryOpFindPeer = "find_peer"
+ directoryOpPlace = "place"
+
+ directoryResultOK = "ok"
+ directoryResultEmpty = "empty"
+ directoryResultInvalid = "invalid"
+ directoryResultError = "error"
+)
+
+var directoryRequestsTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_router_directory_requests_total",
+ Help: "Directory requests served to members and routers, by operation and result; place is a record another router put here",
+ },
+ []string{"op", "result"},
+)
+
+var directoryRequestSeconds = promauto.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Name: "agentmesh_router_directory_request_seconds",
+ Help: "Time to answer a directory request, the lookup across the routers included",
+ Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+ },
+ []string{"op"},
+)
+
+var directoryPlacementsTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_router_directory_placements_total",
+ Help: "Copies of a member's record this router placed on the routers closest to the key, by result",
+ },
+ []string{"result"},
+)
+
+func init() {
+ for _, op := range []string{directoryOpProvide, directoryOpFindProviders, directoryOpFindPeer, directoryOpPlace} {
+ for _, result := range []string{directoryResultOK, directoryResultEmpty, directoryResultInvalid, directoryResultError} {
+ directoryRequestsTotal.WithLabelValues(op, result)
+ }
+ }
+ for _, result := range []string{directoryResultOK, directoryResultError} {
+ directoryPlacementsTotal.WithLabelValues(result)
+ }
+}
+
+// handleDirectory serves one DirectoryRequest on a stream from a member
+// or a router. It is registered through the auth gate, so the peer has
+// authenticated by the time it runs. A member keeps no DHT: it announces
+// its records here and the router keeps them and places them on the
+// routers closest to the key; it looks keys up here and the router runs
+// the lookup across the routers' DHT.
+func (r *Router) handleDirectory(s network.Stream) {
+ defer func() { _ = s.Close() }()
+ remote := s.Conn().RemotePeer()
+ _ = s.SetDeadline(time.Now().Add(directoryRequestTimeout))
+ ctx, cancel := context.WithTimeout(r.lifetime(), directoryRequestTimeout)
+ defer cancel()
+
+ var req api.DirectoryRequest
+ reader := msgio.NewVarintReaderSize(s, directoryMaxMessage)
+ frame, err := reader.ReadMsg()
+ if err != nil {
+ logger.Debugf("[Directory] Request from %s not read: %v", remote, err)
+ _ = s.Reset()
+ return
+ }
+ err = proto.Unmarshal(frame, &req)
+ reader.ReleaseMsg(frame)
+ if err != nil {
+ logger.Warnf("[Directory] Request from %s not decoded: %v", remote, err)
+ directoryRequestsTotal.WithLabelValues("unknown", directoryResultInvalid).Inc()
+ _ = s.Reset()
+ return
+ }
+
+ op, resp := r.serveDirectory(ctx, remote, &req)
+ data, err := proto.Marshal(resp)
+ if err != nil {
+ logger.Errorf("[Directory] Response to %s not encoded: %v", remote, err)
+ _ = s.Reset()
+ return
+ }
+ if err := msgio.NewVarintWriter(s).WriteMsg(data); err != nil {
+ logger.Debugf("[Directory] Response to %s (%s) not written: %v", remote, op, err)
+ _ = s.Reset()
+ }
+}
+
+// serveDirectory answers one request and accounts for it.
+func (r *Router) serveDirectory(ctx context.Context, remote peer.ID, req *api.DirectoryRequest) (string, *api.DirectoryResponse) {
+ start := time.Now()
+ var op string
+ var resp *api.DirectoryResponse
+ var err error
+ switch q := req.GetRequest().(type) {
+ case *api.DirectoryRequest_Provide:
+ op = directoryOpProvide
+ if q.Provide.GetProvider() != "" {
+ op = directoryOpPlace
+ }
+ resp, err = r.directoryProvide(ctx, remote, q.Provide)
+ case *api.DirectoryRequest_FindProviders:
+ op = directoryOpFindProviders
+ resp, err = r.directoryFindProviders(ctx, q.FindProviders)
+ case *api.DirectoryRequest_FindPeer:
+ op = directoryOpFindPeer
+ resp, err = r.directoryFindPeer(ctx, q.FindPeer)
+ default:
+ return "unknown", r.directoryRefused("unknown", remote, errInvalidDirectoryRequest)
+ }
+ directoryRequestSeconds.WithLabelValues(op).Observe(time.Since(start).Seconds())
+ if err != nil {
+ return op, r.directoryRefused(op, remote, err)
+ }
+ result := directoryResultOK
+ if len(resp.GetProviders()) == 0 && resp.GetPeer() == nil && op != directoryOpProvide && op != directoryOpPlace {
+ result = directoryResultEmpty
+ }
+ directoryRequestsTotal.WithLabelValues(op, result).Inc()
+ return op, resp
+}
+
+var errInvalidDirectoryRequest = errors.New("request is not provide, find_providers or find_peer")
+
+// directoryRefused answers a request the router did not run, and says why
+// at the level the cause deserves: a malformed request is the peer's
+// doing, a failed lookup is the mesh's.
+func (r *Router) directoryRefused(op string, remote peer.ID, err error) *api.DirectoryResponse {
+ var invalid *directoryInvalidError
+ if errors.As(err, &invalid) {
+ directoryRequestsTotal.WithLabelValues(op, directoryResultInvalid).Inc()
+ logger.Warnf("[Directory] Refusing %s from %s: %v", op, remote, err)
+ } else {
+ directoryRequestsTotal.WithLabelValues(op, directoryResultError).Inc()
+ logger.Warnf("[Directory] %s from %s failed: %v", op, remote, err)
+ }
+ return &api.DirectoryResponse{Error: err.Error()}
+}
+
+// directoryInvalidError is a request the peer got wrong.
+type directoryInvalidError struct{ err error }
+
+func (e *directoryInvalidError) Error() string { return e.err.Error() }
+func (e *directoryInvalidError) Unwrap() error { return e.err }
+
+func invalidDirectoryRequest(format string, args ...any) error {
+ return &directoryInvalidError{err: fmt.Errorf(format, args...)}
+}
+
+// directoryProvide stores a record. From a member it is the member's own,
+// kept here and placed on the routers closest to the key; from a router it
+// is a member's record that router placed here, which only a router may do.
+func (r *Router) directoryProvide(ctx context.Context, remote peer.ID, q *api.DirectoryProvide) (*api.DirectoryResponse, error) {
+ if err := api.ValidateDirectoryKey(q.GetKey()); err != nil {
+ return nil, invalidDirectoryRequest("%w", err)
+ }
+ provider := remote
+ placed := q.GetProvider() != ""
+ if placed {
+ if !r.isAuthenticatedPeerRouter(remote) {
+ return nil, invalidDirectoryRequest("only a router may name a provider; %s is not one", remote)
+ }
+ p, err := peer.Decode(q.GetProvider())
+ if err != nil {
+ return nil, invalidDirectoryRequest("provider: %w", err)
+ }
+ provider = p
+ }
+ addrs, err := recordAddrs(provider, q.GetAddrs())
+ if err != nil {
+ return nil, invalidDirectoryRequest("%w", err)
+ }
+ if r.DHT == nil {
+ return nil, errors.New("the directory is not up")
+ }
+ if err := r.DHT.ProviderStore().AddProvider(ctx, q.GetKey(), peer.AddrInfo{ID: provider, Addrs: addrs}); err != nil {
+ return nil, fmt.Errorf("store record: %w", err)
+ }
+ if !placed {
+ r.placeRecord(ctx, q.GetKey(), provider, q.GetAddrs())
+ }
+ return &api.DirectoryResponse{}, nil
+}
+
+// recordAddrs parses a record's addresses: at most DirectoryMaxAddrs, each
+// with no peer suffix or the provider's own, stored without it as the
+// peerstore keeps them.
+func recordAddrs(provider peer.ID, raw []string) ([]multiaddr.Multiaddr, error) {
+ if len(raw) > api.DirectoryMaxAddrs {
+ return nil, fmt.Errorf("%d addresses, at most %d", len(raw), api.DirectoryMaxAddrs)
+ }
+ addrs := make([]multiaddr.Multiaddr, 0, len(raw))
+ for _, a := range raw {
+ ma, err := multiaddr.NewMultiaddr(a)
+ if err != nil {
+ return nil, fmt.Errorf("address %q: %w", a, err)
+ }
+ transport, p := peer.SplitAddr(ma)
+ if p != "" && p != provider {
+ return nil, fmt.Errorf("address %q names %s, the record is %s's", a, p, provider)
+ }
+ if transport == nil {
+ return nil, fmt.Errorf("address %q has no transport", a)
+ }
+ addrs = append(addrs, transport)
+ }
+ return addrs, nil
+}
+
+// placeRecord puts a member's record on the routers closest to the key,
+// as many as the mesh's replication, over the directory protocol with the
+// provider named. Every copy is attempted; a router that fails to take it
+// is logged and counted, the member's own router keeps a copy regardless.
+func (r *Router) placeRecord(ctx context.Context, key []byte, provider peer.ID, addrs []string) {
+ targets := r.DHT.RoutingTable().NearestPeers(kb.ConvertKey(string(key)), r.directory.Replication)
+ req := &api.DirectoryRequest{Request: &api.DirectoryRequest_Provide{Provide: &api.DirectoryProvide{
+ Key: key, Addrs: addrs, Provider: provider.String(),
+ }}}
+ data, err := proto.Marshal(req)
+ if err != nil {
+ logger.Errorf("[Directory] Placement for %s not encoded: %v", provider, err)
+ return
+ }
+ var wg sync.WaitGroup
+ for _, target := range targets {
+ wg.Add(1)
+ go func(target peer.ID) {
+ defer wg.Done()
+ if err := r.sendDirectory(ctx, target, data); err != nil {
+ directoryPlacementsTotal.WithLabelValues(directoryResultError).Inc()
+ logger.Warnf("[Directory] Record of %s not placed on router %s: %v", provider, target, err)
+ return
+ }
+ directoryPlacementsTotal.WithLabelValues(directoryResultOK).Inc()
+ }(target)
+ }
+ wg.Wait()
+}
+
+// sendDirectory sends one encoded request to a peer router and reads its
+// answer, over the gated host so the router has authenticated first.
+func (r *Router) sendDirectory(ctx context.Context, target peer.ID, data []byte) error {
+ s, err := r.gated.NewStream(ctx, target, api.DirectoryProtocolID)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = s.Close() }()
+ if deadline, ok := ctx.Deadline(); ok {
+ _ = s.SetDeadline(deadline)
+ }
+ if err := msgio.NewVarintWriter(s).WriteMsg(data); err != nil {
+ _ = s.Reset()
+ return err
+ }
+ reader := msgio.NewVarintReaderSize(s, directoryMaxMessage)
+ frame, err := reader.ReadMsg()
+ if err != nil {
+ _ = s.Reset()
+ return err
+ }
+ defer reader.ReleaseMsg(frame)
+ var resp api.DirectoryResponse
+ if err := proto.Unmarshal(frame, &resp); err != nil {
+ return err
+ }
+ if resp.GetError() != "" {
+ return errors.New(resp.GetError())
+ }
+ return nil
+}
+
+// directoryFindProviders runs the lookup across the routers' DHT.
+func (r *Router) directoryFindProviders(ctx context.Context, q *api.DirectoryFindProviders) (*api.DirectoryResponse, error) {
+ if err := api.ValidateDirectoryKey(q.GetKey()); err != nil {
+ return nil, invalidDirectoryRequest("%w", err)
+ }
+ limit := int(q.GetLimit())
+ if limit <= 0 {
+ limit = directoryDefaultLimit
+ }
+ if limit > directoryMaxLimit {
+ limit = directoryMaxLimit
+ }
+ records, err := r.lookupProviders(ctx, q.GetKey(), limit)
+ if err != nil {
+ return nil, err
+ }
+ return &api.DirectoryResponse{Providers: records}, nil
+}
+
+// lookupProviders is the lookup both find_providers and find_peer run.
+func (r *Router) lookupProviders(ctx context.Context, key []byte, limit int) ([]*api.PeerRecord, error) {
+ if r.DHT == nil {
+ return nil, errors.New("the directory is not up")
+ }
+ c := cid.NewCidV1(cid.Raw, key)
+ records := make([]*api.PeerRecord, 0, limit)
+ seen := map[peer.ID]bool{}
+ for info := range r.DHT.FindProvidersAsync(ctx, c, limit) {
+ if seen[info.ID] {
+ continue
+ }
+ seen[info.ID] = true
+ records = append(records, peerRecord(info))
+ if len(records) >= limit {
+ break
+ }
+ }
+ if err := ctx.Err(); err != nil && len(records) == 0 {
+ return nil, fmt.Errorf("lookup: %w", err)
+ }
+ return records, nil
+}
+
+// directoryFindPeer answers where a peer is reached: from this router's
+// own peerstore for a member attached here, from the record the member
+// announced under its peer key otherwise.
+func (r *Router) directoryFindPeer(ctx context.Context, q *api.DirectoryFindPeer) (*api.DirectoryResponse, error) {
+ p, err := peer.Decode(q.GetPeerId())
+ if err != nil {
+ return nil, invalidDirectoryRequest("peer_id: %w", err)
+ }
+ if r.isPeerAuthenticated(p) && r.Host.Network().Connectedness(p) == network.Connected {
+ if addrs := r.Host.Peerstore().Addrs(p); len(addrs) > 0 {
+ return &api.DirectoryResponse{Peer: peerRecord(peer.AddrInfo{ID: p, Addrs: addrs})}, nil
+ }
+ }
+ records, err := r.lookupProviders(ctx, api.PeerKey(p), 1)
+ if err != nil {
+ return nil, err
+ }
+ for _, rec := range records {
+ if rec.GetPeerId() == p.String() {
+ return &api.DirectoryResponse{Peer: rec}, nil
+ }
+ }
+ return &api.DirectoryResponse{}, nil
+}
+
+// peerRecord renders a peer and its addresses, each with the peer suffix.
+func peerRecord(info peer.AddrInfo) *api.PeerRecord {
+ rec := &api.PeerRecord{PeerId: info.ID.String()}
+ for _, a := range info.Addrs {
+ if _, p := peer.SplitAddr(a); p == "" {
+ a = a.Encapsulate(multiaddr.StringCast("/p2p/" + info.ID.String()))
+ }
+ rec.Addrs = append(rec.Addrs, a.String())
+ }
+ return rec
+}
diff --git a/internal/router/directory_test.go b/internal/router/directory_test.go
new file mode 100644
index 00000000..effb2dfa
--- /dev/null
+++ b/internal/router/directory_test.go
@@ -0,0 +1,280 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "context"
+ "slices"
+ "testing"
+ "time"
+
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ kb "github.com/libp2p/go-libp2p-kbucket"
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-msgio"
+ dto "github.com/prometheus/client_model/go"
+ "google.golang.org/protobuf/proto"
+
+ "github.com/google/agentmesh/api"
+)
+
+// newDirectoryRouter is a router with its DHT and the directory handler
+// up, as Start wires them, on a loopback host.
+func newDirectoryRouter(t *testing.T) *Router {
+ t.Helper()
+ r, gated := newGatedRouter(t)
+ r.gated = gated
+ kdht, err := dht.New(gated, r.dhtOptions()...)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = kdht.Close() })
+ r.DHT = kdht
+ gated.SetStreamHandler(api.DirectoryProtocolID, r.handleDirectory)
+ return r
+}
+
+// federate connects two routers and has each admit the other as an
+// authenticated peer router, as the handshake does.
+func federate(t *testing.T, a, b *Router) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := a.Host.Connect(ctx, peer.AddrInfo{ID: b.Host.ID(), Addrs: b.Host.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+ for _, pair := range [][2]*Router{{a, b}, {b, a}} {
+ pair[0].authenticatedPeers.Store(pair[1].Host.ID(), time.Now().Add(time.Hour))
+ pair[0].admitPeerRouter(pair[1].Host.ID())
+ }
+}
+
+// attach connects a member host to a router and marks it authenticated there.
+func attach(t *testing.T, r *Router) host.Host {
+ t.Helper()
+ m := newClient(t, r)
+ r.authenticatedPeers.Store(m.ID(), time.Now().Add(time.Hour))
+ return m
+}
+
+// ask sends one request to a router and returns its answer.
+func ask(t *testing.T, from host.Host, router peer.ID, req *api.DirectoryRequest) *api.DirectoryResponse {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ s, err := from.NewStream(ctx, router, api.DirectoryProtocolID)
+ if err != nil {
+ t.Fatalf("open directory stream: %v", err)
+ }
+ defer func() { _ = s.Close() }()
+ data, err := proto.Marshal(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := msgio.NewVarintWriter(s).WriteMsg(data); err != nil {
+ t.Fatal(err)
+ }
+ frame, err := msgio.NewVarintReaderSize(s, directoryMaxMessage).ReadMsg()
+ if err != nil {
+ t.Fatalf("read directory answer: %v", err)
+ }
+ var resp api.DirectoryResponse
+ if err := proto.Unmarshal(frame, &resp); err != nil {
+ t.Fatal(err)
+ }
+ return &resp
+}
+
+func provide(key []byte, addrs ...string) *api.DirectoryRequest {
+ return &api.DirectoryRequest{Request: &api.DirectoryRequest_Provide{Provide: &api.DirectoryProvide{Key: key, Addrs: addrs}}}
+}
+
+func findProviders(key []byte) *api.DirectoryRequest {
+ return &api.DirectoryRequest{Request: &api.DirectoryRequest_FindProviders{FindProviders: &api.DirectoryFindProviders{Key: key}}}
+}
+
+func findPeer(p peer.ID) *api.DirectoryRequest {
+ return &api.DirectoryRequest{Request: &api.DirectoryRequest_FindPeer{FindPeer: &api.DirectoryFindPeer{PeerId: p.String()}}}
+}
+
+func directoryCount(t *testing.T, op, result string) float64 {
+ t.Helper()
+ var m dto.Metric
+ if err := directoryRequestsTotal.WithLabelValues(op, result).Write(&m); err != nil {
+ t.Fatal(err)
+ }
+ return m.GetCounter().GetValue()
+}
+
+// A member announces a record to the router it is attached to; the router
+// keeps it and places it on the routers closest to the key, as many as the
+// mesh's replication. A member attached to any router then finds it: from
+// that router's own store when it holds a copy, through the routers' DHT
+// otherwise. A peer is found the same way, under its peer key, or from the
+// router that holds it.
+func TestDirectoryRecordsAreFoundFromAnyRouter(t *testing.T) {
+ a, b, c := newDirectoryRouter(t), newDirectoryRouter(t), newDirectoryRouter(t)
+ federate(t, a, b)
+ federate(t, a, c)
+ federate(t, b, c)
+ // One copy besides the member's own router, on the closest of the others.
+ a.directory.Replication = 1
+
+ member := attach(t, a)
+ key, err := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "calculator")
+ if err != nil {
+ t.Fatal(err)
+ }
+ memberAddr := member.Addrs()[0].String() + "/p2p/" + member.ID().String()
+ if resp := ask(t, member, a.Host.ID(), provide(key, memberAddr)); resp.GetError() != "" {
+ t.Fatalf("provide refused: %s", resp.GetError())
+ }
+ if resp := ask(t, member, a.Host.ID(), provide(api.PeerKey(member.ID()), memberAddr)); resp.GetError() != "" {
+ t.Fatalf("provide of the peer key refused: %s", resp.GetError())
+ }
+
+ closest := a.DHT.RoutingTable().NearestPeers(kb.ConvertKey(string(key)), 1)
+ if len(closest) != 1 {
+ t.Fatalf("closest routers to the key = %v, want one", closest)
+ }
+ holder, other := b, c
+ if closest[0] == c.Host.ID() {
+ holder, other = c, b
+ }
+ if provs, _ := holder.DHT.ProviderStore().GetProviders(context.Background(), key); len(provs) != 1 || provs[0].ID != member.ID() {
+ t.Fatalf("the closest router holds %v, want the member's record", provs)
+ }
+ if provs, _ := other.DHT.ProviderStore().GetProviders(context.Background(), key); len(provs) != 0 {
+ t.Fatalf("the other router holds %v, want nothing: replication is one", provs)
+ }
+
+ for name, router := range map[string]*Router{"own router": a, "holder": holder, "other router, through the DHT": other} {
+ t.Run("find providers from the "+name, func(t *testing.T) {
+ asker := attach(t, router)
+ resp := ask(t, asker, router.Host.ID(), findProviders(key))
+ if resp.GetError() != "" {
+ t.Fatalf("find_providers failed: %s", resp.GetError())
+ }
+ if len(resp.GetProviders()) != 1 || resp.GetProviders()[0].GetPeerId() != member.ID().String() {
+ t.Fatalf("providers = %v, want the member", resp.GetProviders())
+ }
+ if got := resp.GetProviders()[0].GetAddrs(); !slices.Contains(got, memberAddr) {
+ t.Fatalf("provider addrs = %v, want %s among them", got, memberAddr)
+ }
+ })
+ }
+
+ t.Run("find peer from the other router", func(t *testing.T) {
+ asker := attach(t, other)
+ resp := ask(t, asker, other.Host.ID(), findPeer(member.ID()))
+ if resp.GetError() != "" {
+ t.Fatalf("find_peer failed: %s", resp.GetError())
+ }
+ if resp.GetPeer().GetPeerId() != member.ID().String() || !slices.Contains(resp.GetPeer().GetAddrs(), memberAddr) {
+ t.Fatalf("peer = %v, want the member at %s", resp.GetPeer(), memberAddr)
+ }
+ })
+
+ t.Run("find peer from its own router answers from the connection", func(t *testing.T) {
+ asker := attach(t, a)
+ resp := ask(t, asker, a.Host.ID(), findPeer(member.ID()))
+ if resp.GetPeer().GetPeerId() != member.ID().String() || len(resp.GetPeer().GetAddrs()) == 0 {
+ t.Fatalf("peer = %v, want the member with addresses", resp.GetPeer())
+ }
+ })
+
+ t.Run("nobody is found under a key nobody announced", func(t *testing.T) {
+ before := directoryCount(t, directoryOpFindProviders, directoryResultEmpty)
+ unknown, _ := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "nobody")
+ asker := attach(t, b)
+ resp := ask(t, asker, b.Host.ID(), findProviders(unknown))
+ if resp.GetError() != "" || len(resp.GetProviders()) != 0 {
+ t.Fatalf("got %v, want an empty answer and no error", resp)
+ }
+ if got := directoryCount(t, directoryOpFindProviders, directoryResultEmpty); got != before+1 {
+ t.Fatalf("empty answers counted = %v, want %v", got, before+1)
+ }
+ if resp := ask(t, asker, b.Host.ID(), findPeer(asker.ID())); resp.GetPeer() != nil && len(resp.GetPeer().GetAddrs()) == 0 {
+ t.Fatalf("find_peer of a peer with no record = %v, want unset or its own connection", resp.GetPeer())
+ }
+ })
+}
+
+// A member can only announce itself: a record naming another provider, an
+// address of another peer, or a key that is not a sha256 multihash is
+// refused and counted as invalid, and nothing is stored.
+func TestDirectoryRefusesWhatAMemberMayNotAnnounce(t *testing.T) {
+ a := newDirectoryRouter(t)
+ member := attach(t, a)
+ stranger := newClient(t, a)
+ key, _ := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "calculator")
+
+ for name, req := range map[string]*api.DirectoryRequest{
+ "another provider": {Request: &api.DirectoryRequest_Provide{Provide: &api.DirectoryProvide{Key: key, Provider: stranger.ID().String()}}},
+ "another peer's address": provide(key, stranger.Addrs()[0].String()+"/p2p/"+stranger.ID().String()),
+ "a key that is not a multihash": provide([]byte("calculator")),
+ "a lookup of such a key": findProviders([]byte("calculator")),
+ "a peer id that is not one": findPeer(peer.ID("nobody")),
+ } {
+ t.Run(name, func(t *testing.T) {
+ op := directoryOpProvide
+ switch req.GetRequest().(type) {
+ case *api.DirectoryRequest_FindProviders:
+ op = directoryOpFindProviders
+ case *api.DirectoryRequest_FindPeer:
+ op = directoryOpFindPeer
+ }
+ if req.GetProvide().GetProvider() != "" {
+ op = directoryOpPlace
+ }
+ before := directoryCount(t, op, directoryResultInvalid)
+ resp := ask(t, member, a.Host.ID(), req)
+ if resp.GetError() == "" {
+ t.Fatalf("accepted; want a refusal")
+ }
+ if got := directoryCount(t, op, directoryResultInvalid); got != before+1 {
+ t.Fatalf("invalid %s counted = %v, want %v", op, got, before+1)
+ }
+ })
+ }
+ if provs, _ := a.DHT.ProviderStore().GetProviders(context.Background(), key); len(provs) != 0 {
+ t.Fatalf("a refused record was stored: %v", provs)
+ }
+}
+
+// A record placed by a router names its provider; the receiving router
+// stores it under that provider, so a lookup there returns the member and
+// not the router that placed it.
+func TestDirectoryPlacementNamesTheProvider(t *testing.T) {
+ a, b := newDirectoryRouter(t), newDirectoryRouter(t)
+ federate(t, a, b)
+ member := attach(t, a)
+ key, _ := api.ServiceKey(api.ServiceType_SERVICE_TYPE_INFERENCE, "")
+ memberAddr := member.Addrs()[0].String() + "/p2p/" + member.ID().String()
+ if resp := ask(t, member, a.Host.ID(), provide(key, memberAddr)); resp.GetError() != "" {
+ t.Fatalf("provide refused: %s", resp.GetError())
+ }
+ provs, err := b.DHT.ProviderStore().GetProviders(context.Background(), key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(provs) != 1 || provs[0].ID != member.ID() {
+ t.Fatalf("router b holds %v, want the member's record placed by router a", provs)
+ }
+ if len(provs[0].Addrs) == 0 || provs[0].Addrs[0].String() != member.Addrs()[0].String() {
+ t.Fatalf("router b holds addrs %v, want the member's %v without the peer suffix", provs[0].Addrs, member.Addrs()[0])
+ }
+}
diff --git a/internal/router/resource_limits.go b/internal/router/resource_limits.go
index 4dc6f1a7..594f1684 100644
--- a/internal/router/resource_limits.go
+++ b/internal/router/resource_limits.go
@@ -38,7 +38,7 @@ import (
// discovery (libp2p_rcmgr_blocked_resources{scope="protocol"} counted them,
// without saying which protocol).
var memberProtocols = slices.Concat(
- []protocol.ID{identify.ID, identify.IDPush, api.AuthProtocolID, api.DHTProtocolID},
+ []protocol.ID{identify.ID, identify.IDPush, api.AuthProtocolID, api.DHTProtocolID, api.DirectoryProtocolID},
pubsub.GossipSubDefaultProtocols,
)
diff --git a/internal/router/router.go b/internal/router/router.go
index 42face05..1871bd70 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -338,6 +338,9 @@ type Router struct {
// peerRouters is the other routers of the mesh, as /info lists them or as
// a handshake proved them: held whatever the load and never sent away.
peerRouters sync.Map
+ // gated is the host the DHT, gossip and the directory run on: it serves
+ // and reaches authenticated peers only (authGate).
+ gated *authGate
// handshakeLimiter bounds /mesh/auth attempts per peer; any internet peer
// can open those streams.
handshakeLimiter *ratelimit.PeerRateLimiter
@@ -505,8 +508,10 @@ func (r *Router) Start() (retErr error) {
return err
}
r.Host = hostNode
- // The DHT and gossip serve, and are served by, authenticated peers only.
+ // The DHT, gossip and the directory serve, and are served by,
+ // authenticated peers only.
gated := &authGate{Host: hostNode, r: r}
+ r.gated = gated
// Setup DHT
kadDHT, err := dht.New(gated, r.dhtOptions()...)
@@ -551,6 +556,9 @@ func (r *Router) Start() (retErr error) {
// Setup authentication stream handler
hostNode.SetStreamHandler(api.AuthProtocolID, recoverStreamHandler("AuthHandshake", r.HandleAuthHandshake))
+ // The directory a member asks this router for, and routers place
+ // records on each other through.
+ gated.SetStreamHandler(api.DirectoryProtocolID, recoverStreamHandler("Directory", r.handleDirectory))
// Clean authenticated status on peer disconnection
hostNode.Network().Notify(&network.NotifyBundle{