Skip to content

Agent Mesh

node: ask the routers for the directory instead of keeping a DHT

Antonio Ojeacb46231183a8
diff --git a/go.mod b/go.mod
index a087e4ff..58fcd9a7 100644
--- a/go.mod
+++ b/go.mod
@@ -24,7 +24,6 @@ require (
github.com/modelcontextprotocol/go-sdk v1.8.0
github.com/multiformats/go-multiaddr v0.16.1
github.com/multiformats/go-multiaddr-dns v0.6.0
- github.com/multiformats/go-multihash v0.2.3
github.com/openai/openai-go v1.12.0
github.com/prometheus/client_golang v1.24.1
github.com/prometheus/client_model v0.6.3
@@ -96,6 +95,7 @@ require (
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multibase v0.3.0 // indirect
github.com/multiformats/go-multicodec v0.10.0 // indirect
+ github.com/multiformats/go-multihash v0.2.3 // indirect
github.com/multiformats/go-multistream v0.6.1 // indirect
github.com/multiformats/go-varint v0.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
diff --git a/internal/node/auth_gate.go b/internal/node/auth_gate.go
index baa410d7..ee35d8a1 100644
--- a/internal/node/auth_gate.go
+++ b/internal/node/auth_gate.go
@@ -107,6 +107,18 @@ func (n *AgentMeshNode) requireAuthenticated(next network.StreamHandler) network
}
}
+// isCatalogRouter reports whether p is a router the control plane lists.
+func (n *AgentMeshNode) isCatalogRouter(p peer.ID) bool {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ for _, c := range n.routerCatalog {
+ if c.ID == p {
+ return true
+ }
+ }
+ return false
+}
+
// awaitPeerAuthenticated returns once p is authenticated, or with an error
// when ctx ends, p's connections are gone, or authWait passes.
func (n *AgentMeshNode) awaitPeerAuthenticated(ctx context.Context, p peer.ID) error {
diff --git a/internal/node/debug_handlers.go b/internal/node/debug_handlers.go
index f9b05cac..765bcd91 100644
--- a/internal/node/debug_handlers.go
+++ b/internal/node/debug_handlers.go
@@ -109,8 +109,8 @@ func (n *AgentMeshNode) debugReady() error {
return fmt.Errorf("node not initialized")
case n.Host == nil:
return fmt.Errorf("libp2p host not initialized")
- case n.DHT == nil:
- return fmt.Errorf("DHT not initialized")
+ case n.Directory == nil:
+ return fmt.Errorf("the directory is not up")
case n.Store == nil:
return fmt.Errorf("node store not initialized")
}
@@ -132,12 +132,8 @@ type meshInfoResponse struct {
PeerID string `json:"peer_id"`
ConnectedPeers []string `json:"connected_peers"`
AttachedRouters []string `json:"attached_routers"`
- // DirectoryRouters are the routers the node holds a session with for
- // its DHT alone: the ones its keys landed on, not attached.
- DirectoryRouters []string `json:"directory_routers"`
- DHTSize int `json:"dht_size"`
- RouterPeerID string `json:"router_peer_id"`
- LocalAPISocket string `json:"local_api_socket,omitempty"`
+ RouterPeerID string `json:"router_peer_id"`
+ LocalAPISocket string `json:"local_api_socket,omitempty"`
}
type connectivityResponse struct {
@@ -194,21 +190,13 @@ func (n *AgentMeshNode) meshInfo() (*meshInfoResponse, error) {
attachedRouters = append(attachedRouters, p.String())
}
slices.Sort(attachedRouters)
- directory := n.directoryRouters()
- directoryRouters := make([]string, 0, len(directory))
- for _, p := range directory {
- directoryRouters = append(directoryRouters, p.String())
- }
- slices.Sort(directoryRouters)
return &meshInfoResponse{
- PeerID: n.Host.ID().String(),
- ConnectedPeers: connectedPeers,
- AttachedRouters: attachedRouters,
- DirectoryRouters: directoryRouters,
- DHTSize: n.DHT.RoutingTable().Size(),
- RouterPeerID: n.RouterPeerID.String(),
- LocalAPISocket: n.BoundSocketPath,
+ PeerID: n.Host.ID().String(),
+ ConnectedPeers: connectedPeers,
+ AttachedRouters: attachedRouters,
+ RouterPeerID: n.RouterPeerID.String(),
+ LocalAPISocket: n.BoundSocketPath,
}, nil
}
@@ -221,7 +209,7 @@ func (n *AgentMeshNode) peerAddrs(ctx context.Context, pid peer.ID) (*peerAddrsR
}
ctx, cancel := context.WithTimeout(ctx, dhtLookupTimeout)
defer cancel()
- info, err := n.DHT.FindPeer(ctx, pid)
+ info, err := n.Directory.FindPeer(ctx, pid)
if err != nil {
return nil, fmt.Errorf("find %s: %w", pid, err)
}
diff --git a/internal/node/dht.go b/internal/node/dht.go
deleted file mode 100644
index 4add2d56..00000000
--- a/internal/node/dht.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package node
-
-import (
- dht "github.com/libp2p/go-libp2p-kad-dht"
- "github.com/libp2p/go-libp2p-kad-dht/records"
-
- "github.com/google/agentmesh/api"
-)
-
-// dhtOptions configures the member's DHT. A member is a DHT client whatever
-// its reachability: the routers are the directory, and a member that served
-// it would sit in every other member's routing table and take their lookups
-// on a 400-connection budget, answer whoever connects, and be trusted for
-// unsigned provider records. A public member is still dialled directly; the
-// role in the directory and the way it is reached are separate things.
-//
-// The table is not refreshed. kad-dht treats a table of ten peers or fewer
-// as starved and, every two minutes, pings every entry and runs a lookup
-// for the member's own ID against all of them; with the routers as the only
-// servers that is a connection to every router the member is not attached
-// to, closed by the router at its auth deadline and opened again two
-// minutes later, for nothing the member needs: the servers are the routers
-// the control plane lists, learned on attach and from closer-peer answers,
-// and a router that leaves the catalog leaves the table (setRouterCatalog).
-// Bootstrap still fills the table once at start.
-//
-// The record TTL and the replication factor are the mesh's
-// (ControlPlaneInfoResponse.directory). The TTL is the same figure the
-// routers keep a record for, so a record the member's own store holds and
-// one a router holds expire alike. The replication factor r is kademlia's
-// k on the member: a record is stored on the r routers closest to its key,
-// a lookup asks those r, r at a time. The routers keep every router in
-// their own tables (router.dhtOptions) and answer FIND_NODE with the
-// closest routers to the key in order, so a member learns the r closest
-// from one answer of a router it is attached to and reaches them in one
-// hop. A member therefore holds sessions with its K attached routers plus
-// the r closest to each key it publishes or looks up, whatever the number
-// of routers; with the library's k of 20 that would be every router of a
-// mesh with fewer than 20.
-func dhtOptions(directory api.DirectorySettings) []dht.Option {
- return []dht.Option{
- dht.Mode(dht.ModeClient),
- dht.DisableAutoRefresh(),
- dht.ProtocolPrefix(api.DHTProtocolPrefix),
- dht.BucketSize(directory.Replication),
- dht.Concurrency(directory.Replication),
- dht.Resiliency(directory.Replication),
- dht.ProviderManagerOpts(records.ProviderAddrTTL(directory.RecordTTL), records.ProvideValidity(directory.RecordTTL)),
- }
-}
diff --git a/internal/node/dht_test.go b/internal/node/dht_test.go
deleted file mode 100644
index 0b3b43e8..00000000
--- a/internal/node/dht_test.go
+++ /dev/null
@@ -1,161 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package node
-
-import (
- "context"
- "fmt"
- "slices"
- "testing"
- "time"
-
- "github.com/libp2p/go-libp2p"
- dht "github.com/libp2p/go-libp2p-kad-dht"
- "github.com/libp2p/go-libp2p/core/host"
- "github.com/libp2p/go-libp2p/core/peer"
- "github.com/libp2p/go-libp2p/core/protocol"
- "github.com/libp2p/go-libp2p/core/test"
-
- "github.com/google/agentmesh/api"
-)
-
-// TestMemberDHTStaysClientWhenPublic: a member whose reachability is public
-// must not serve the DHT protocol, or it lands in every other member's
-// routing table. The mode is the decision; the protocol list is what other
-// members see of it. The control DHT, in the mode the node used before,
-// shows that this host setup does flip a ModeAuto DHT to server.
-func TestMemberDHTStaysClientWhenPublic(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
- t.Cleanup(cancel)
-
- newHost := func() host.Host {
- h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"), libp2p.ForceReachabilityPublic())
- if err != nil {
- t.Fatalf("libp2p.New: %v", err)
- }
- t.Cleanup(func() { _ = h.Close() })
- return h
- }
- serves := func(h host.Host) bool {
- return slices.Contains(h.Mux().Protocols(), protocol.ID(api.DHTProtocolID))
- }
-
- control := newHost()
- controlDHT, err := dht.New(control, dht.Mode(dht.ModeAuto), dht.ProtocolPrefix(api.DHTProtocolPrefix))
- if err != nil {
- t.Fatalf("control dht.New: %v", err)
- }
- t.Cleanup(func() { _ = controlDHT.Close() })
-
- member := newHost()
- memberDHT, err := dht.New(member, dhtOptions(api.DefaultDirectorySettings())...)
- if err != nil {
- t.Fatalf("member dht.New: %v", err)
- }
- t.Cleanup(func() { _ = memberDHT.Close() })
- if err := memberDHT.Bootstrap(ctx); err != nil {
- t.Fatalf("Bootstrap: %v", err)
- }
-
- deadline := time.Now().Add(10 * time.Second)
- for !serves(control) {
- if time.Now().After(deadline) {
- t.Fatalf("control DHT in ModeAuto on a public host never served %s; protocols: %v", api.DHTProtocolID, control.Mux().Protocols())
- }
- time.Sleep(20 * time.Millisecond)
- }
-
- if got := memberDHT.Mode(); got != dht.ModeClient {
- t.Errorf("member DHT mode = %v, want %v", got, dht.ModeClient)
- }
- if serves(member) {
- t.Errorf("member on a public host serves %s; protocols: %v", api.DHTProtocolID, member.Mux().Protocols())
- }
-}
-
-// TestSetRouterCatalogPrunesDHTTable: with kad-dht's liveness pings off, the
-// catalog is what takes a router out of the member's DHT table.
-func TestSetRouterCatalogPrunesDHTTable(t *testing.T) {
- h, err := libp2p.New(libp2p.NoListenAddrs)
- if err != nil {
- t.Fatalf("libp2p.New: %v", err)
- }
- t.Cleanup(func() { _ = h.Close() })
- kdht, err := dht.New(h, dhtOptions(api.DefaultDirectorySettings())...)
- if err != nil {
- t.Fatalf("dht.New: %v", err)
- }
- t.Cleanup(func() { _ = kdht.Close() })
-
- listed, gone := test.RandPeerIDFatal(t), test.RandPeerIDFatal(t)
- for _, p := range []peer.ID{listed, gone} {
- if added, err := kdht.RoutingTable().TryAddPeer(p, false, false); err != nil || !added {
- t.Fatalf("TryAddPeer(%s) = %v, %v; want added", p, added, err)
- }
- }
-
- n := &AgentMeshNode{config: Options{Routers: 2}, Host: h, DHT: kdht}
- n.setRouterCatalog([]routerCandidate{{AddrInfo: peer.AddrInfo{ID: listed}}})
-
- got := kdht.RoutingTable().ListPeers()
- if len(got) != 1 || got[0] != listed {
- t.Fatalf("routing table after the catalog dropped %s = %v, want [%s]", gone, got, listed)
- }
-
- // An empty catalog says nothing about the routers and must not empty the table.
- n.setRouterCatalog(nil)
- if got := kdht.RoutingTable().ListPeers(); len(got) != 1 {
- t.Fatalf("routing table after an empty catalog = %v, want [%s]", got, listed)
- }
-}
-
-// TestMemberDHTBucketHoldsTheReplicationFactor: the mesh's replication
-// factor is kademlia's k on the member, and a bucket of its table holds
-// that many.
-func TestMemberDHTBucketHoldsTheReplicationFactor(t *testing.T) {
- for _, r := range []int{2, api.DefaultDHTReplication, 5} {
- t.Run(fmt.Sprintf("replication=%d", r), func(t *testing.T) {
- h, err := libp2p.New(libp2p.NoListenAddrs)
- if err != nil {
- t.Fatalf("libp2p.New: %v", err)
- }
- t.Cleanup(func() { _ = h.Close() })
- settings := api.DefaultDirectorySettings()
- settings.Replication = r
- kdht, err := dht.New(h, dhtOptions(settings)...)
- if err != nil {
- t.Fatalf("dht.New: %v", err)
- }
- t.Cleanup(func() { _ = kdht.Close() })
- rt := kdht.RoutingTable()
- added := 0
- for i := 0; i < r+1; i++ {
- p, err := rt.GenRandPeerID(0)
- if err != nil {
- t.Fatalf("GenRandPeerID: %v", err)
- }
- // The last one is refused for capacity; any other refusal is a test bug.
- if ok, err := rt.TryAddPeer(p, true, false); ok {
- added++
- } else if added < r {
- t.Fatalf("TryAddPeer #%d refused before the bucket was full: %v", i, err)
- }
- }
- if added != r {
- t.Fatalf("a bucket took %d peers, want %d", added, r)
- }
- })
- }
-}
diff --git a/internal/node/directory.go b/internal/node/directory.go
deleted file mode 100644
index ec44351d..00000000
--- a/internal/node/directory.go
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package node
-
-import (
- "context"
-
- "github.com/libp2p/go-libp2p/core/peer"
-)
-
-// authDirectoryRouter runs the credential handshake with a router this node
-// just connected to without being attached to it. The DHT client opens such
-// connections for a lookup or an announcement, to the routers kademlia puts
-// closest to the key; left anonymous, the router closes them at its auth
-// deadline a minute later and the next lookup opens them again. Handshaken,
-// they are sessions: the router keeps them, counts them, and the node's
-// lookups reuse them. The routers a node holds this way are its K attached
-// ones plus those its keys land on, which is how a mesh with more routers
-// spreads them; a node's own connection manager trims the idle ones first.
-// Attached routers handshake on attach, and a connection that arrives while
-// that handshake is in flight waits for it (authRouter).
-func (n *AgentMeshNode) authDirectoryRouter(ctx context.Context, router peer.ID) {
- if n.isAttached(router) || n.isAuthenticatedAndConnected(router) || !n.isCatalogRouter(router) {
- return
- }
- go func() {
- biscuitBytes, err := n.loadIdentityForAuth()
- if err != nil {
- logger.Debugf("[AuthN] Cannot authenticate to directory router %s without an identity: %v", router, err)
- return
- }
- if err := n.authRouter(ctx, peer.AddrInfo{ID: router}, biscuitBytes); err != nil {
- directoryHandshakesTotal.WithLabelValues("failed").Inc()
- logger.Warnf("[AuthN] Directory router %s did not admit this node; the router will close the connection at its auth deadline: %v", router, err)
- return
- }
- directoryHandshakesTotal.WithLabelValues("ok").Inc()
- logger.Infof("[AuthN] Authenticated to directory router %s", router)
- }()
-}
-
-// isCatalogRouter reports whether p is a router the control plane lists.
-func (n *AgentMeshNode) isCatalogRouter(p peer.ID) bool {
- n.mu.Lock()
- defer n.mu.Unlock()
- for _, c := range n.routerCatalog {
- if c.ID == p {
- return true
- }
- }
- return false
-}
-
-// directoryRouters are the routers this node holds an authenticated session
-// with for its DHT alone: connected and admitted, not attached.
-func (n *AgentMeshNode) directoryRouters() []peer.ID {
- attached := map[peer.ID]bool{}
- n.mu.Lock()
- for r := range n.attached {
- attached[r] = true
- }
- n.mu.Unlock()
- var out []peer.ID
- for _, r := range n.authenticatedRouterIDs() {
- if !attached[r] {
- out = append(out, r)
- }
- }
- return out
-}
-
-// DirectoryRouters is how many routers this node holds for its DHT alone.
-func (n *AgentMeshNode) DirectoryRouters() int { return len(n.directoryRouters()) }
diff --git a/internal/node/directory_client.go b/internal/node/directory_client.go
new file mode 100644
index 00000000..e18a94e1
--- /dev/null
+++ b/internal/node/directory_client.go
@@ -0,0 +1,261 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/peerstore"
+ "github.com/libp2p/go-libp2p/core/routing"
+ "github.com/libp2p/go-msgio"
+ "github.com/multiformats/go-multiaddr"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+ "google.golang.org/protobuf/proto"
+
+ "github.com/google/agentmesh/api"
+)
+
+// Directory is the mesh's directory as a member sees it: where it
+// announces what it serves and where it is reached, and where it looks
+// peers and services up. The routers the member is attached to answer for
+// it over api.DirectoryProtocolID; the member keeps no DHT of its own.
+type Directory interface {
+ // Provide announces this member as a provider of key, at the
+ // addresses it announces, to a router it is attached to; the router
+ // keeps the record and places it on the routers closest to the key.
+ Provide(ctx context.Context, key []byte) error
+ // FindProviders is who provides key, up to limit, with their addresses.
+ FindProviders(ctx context.Context, key []byte, limit int) ([]peer.AddrInfo, error)
+ // FindPeer is where p is reached; routing.ErrNotFound when nobody knows.
+ FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error)
+}
+
+const (
+ // directoryRequestTimeout bounds one request to one router: the router's
+ // lookup across the mesh included, which is one round between routers
+ // that all know each other.
+ directoryRequestTimeout = 10 * time.Second
+ directoryMaxMessage = 64 * 1024
+ directoryDefaultLimit = 20
+)
+
+var errNoDirectoryRouter = errors.New("no attached router to ask")
+
+var directoryRequestsTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_node_directory_requests_total",
+ Help: "Directory requests this node made to its routers, by operation and result; failed is a request no attached router answered",
+ },
+ []string{"op", "result"},
+)
+
+var directoryRequestSeconds = promauto.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Name: "agentmesh_node_directory_request_seconds",
+ Help: "Time a directory request took, the router's lookup included",
+ Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+ },
+ []string{"op"},
+)
+
+const (
+ directoryOpProvide = "provide"
+ directoryOpFindProviders = "find_providers"
+ directoryOpFindPeer = "find_peer"
+
+ directoryResultOK = "ok"
+ directoryResultEmpty = "empty"
+ directoryResultFailed = "failed"
+)
+
+func init() {
+ for _, op := range []string{directoryOpProvide, directoryOpFindProviders, directoryOpFindPeer} {
+ for _, result := range []string{directoryResultOK, directoryResultEmpty, directoryResultFailed} {
+ directoryRequestsTotal.WithLabelValues(op, result)
+ }
+ }
+}
+
+// routerDirectory asks the routers the node is attached to. A request goes
+// to one of them; when it fails, the next is asked, so one router that is
+// restarting costs a request a timeout and not an answer.
+type routerDirectory struct {
+ host host.Host // the gated host: a stream to a router follows its handshake
+ routers func() []peer.ID
+ addrs func() []multiaddr.Multiaddr
+}
+
+func (d *routerDirectory) Provide(ctx context.Context, key []byte) error {
+ addrs := d.addrs()
+ rec := &api.DirectoryProvide{Key: key, Addrs: make([]string, 0, len(addrs))}
+ for _, a := range addrs {
+ rec.Addrs = append(rec.Addrs, a.String())
+ }
+ if len(rec.Addrs) > api.DirectoryMaxAddrs {
+ rec.Addrs = rec.Addrs[:api.DirectoryMaxAddrs]
+ }
+ _, err := d.request(ctx, directoryOpProvide, &api.DirectoryRequest{Request: &api.DirectoryRequest_Provide{Provide: rec}})
+ return err
+}
+
+func (d *routerDirectory) FindProviders(ctx context.Context, key []byte, limit int) ([]peer.AddrInfo, error) {
+ if limit <= 0 {
+ limit = directoryDefaultLimit
+ }
+ resp, err := d.request(ctx, directoryOpFindProviders, &api.DirectoryRequest{Request: &api.DirectoryRequest_FindProviders{
+ FindProviders: &api.DirectoryFindProviders{Key: key, Limit: uint32(limit)},
+ }})
+ if err != nil {
+ return nil, err
+ }
+ out := make([]peer.AddrInfo, 0, len(resp.GetProviders()))
+ for _, rec := range resp.GetProviders() {
+ info, err := addrInfoOf(rec)
+ if err != nil {
+ logger.Warnf("[Directory] Skipping a provider record the router sent: %v", err)
+ continue
+ }
+ out = append(out, info)
+ }
+ return out, nil
+}
+
+func (d *routerDirectory) FindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo, error) {
+ resp, err := d.request(ctx, directoryOpFindPeer, &api.DirectoryRequest{Request: &api.DirectoryRequest_FindPeer{
+ FindPeer: &api.DirectoryFindPeer{PeerId: p.String()},
+ }})
+ if err != nil {
+ return peer.AddrInfo{}, err
+ }
+ if resp.GetPeer() == nil {
+ return peer.AddrInfo{ID: p}, routing.ErrNotFound
+ }
+ info, err := addrInfoOf(resp.GetPeer())
+ if err != nil {
+ return peer.AddrInfo{ID: p}, err
+ }
+ if info.ID != p {
+ return peer.AddrInfo{ID: p}, fmt.Errorf("the router answered for %s", info.ID)
+ }
+ return info, nil
+}
+
+// request sends req to the attached routers in turn until one answers.
+func (d *routerDirectory) request(ctx context.Context, op string, req *api.DirectoryRequest) (*api.DirectoryResponse, error) {
+ start := time.Now()
+ defer func() { directoryRequestSeconds.WithLabelValues(op).Observe(time.Since(start).Seconds()) }()
+ data, err := proto.Marshal(req)
+ if err != nil {
+ return nil, err
+ }
+ routers := d.routers()
+ if len(routers) == 0 {
+ directoryRequestsTotal.WithLabelValues(op, directoryResultFailed).Inc()
+ logger.Warnf("[Directory] %s: %v", op, errNoDirectoryRouter)
+ return nil, errNoDirectoryRouter
+ }
+ var lastErr error
+ for _, router := range routers {
+ resp, err := d.ask(ctx, router, data)
+ if err == nil {
+ result := directoryResultOK
+ if op != directoryOpProvide && len(resp.GetProviders()) == 0 && resp.GetPeer() == nil {
+ result = directoryResultEmpty
+ }
+ directoryRequestsTotal.WithLabelValues(op, result).Inc()
+ return resp, nil
+ }
+ lastErr = err
+ logger.Warnf("[Directory] %s through router %s failed: %v", op, router, err)
+ if ctx.Err() != nil {
+ break
+ }
+ }
+ directoryRequestsTotal.WithLabelValues(op, directoryResultFailed).Inc()
+ return nil, fmt.Errorf("%s: no attached router answered: %w", op, lastErr)
+}
+
+// ask sends one encoded request to one router and reads its answer.
+func (d *routerDirectory) ask(ctx context.Context, router peer.ID, data []byte) (*api.DirectoryResponse, error) {
+ ctx, cancel := context.WithTimeout(ctx, directoryRequestTimeout)
+ defer cancel()
+ s, err := d.host.NewStream(ctx, router, api.DirectoryProtocolID)
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = s.Close() }()
+ if deadline, ok := ctx.Deadline(); ok {
+ _ = s.SetDeadline(deadline)
+ }
+ if err := msgio.NewVarintWriter(s).WriteMsg(data); err != nil {
+ _ = s.Reset()
+ return nil, err
+ }
+ reader := msgio.NewVarintReaderSize(s, directoryMaxMessage)
+ frame, err := reader.ReadMsg()
+ if err != nil {
+ _ = s.Reset()
+ return nil, err
+ }
+ defer reader.ReleaseMsg(frame)
+ var resp api.DirectoryResponse
+ if err := proto.Unmarshal(frame, &resp); err != nil {
+ return nil, err
+ }
+ if resp.GetError() != "" {
+ return nil, errors.New(resp.GetError())
+ }
+ return &resp, nil
+}
+
+// addrInfoOf parses a record into a peer and its transport addresses, the
+// peer suffix off, as the peerstore keeps them; an address naming another
+// peer is the router's mistake and is dropped.
+func addrInfoOf(rec *api.PeerRecord) (peer.AddrInfo, error) {
+ p, err := peer.Decode(rec.GetPeerId())
+ if err != nil {
+ return peer.AddrInfo{}, fmt.Errorf("peer %q: %w", rec.GetPeerId(), err)
+ }
+ info := peer.AddrInfo{ID: p}
+ for _, a := range rec.GetAddrs() {
+ ma, err := multiaddr.NewMultiaddr(a)
+ if err != nil {
+ return peer.AddrInfo{}, fmt.Errorf("address %q of %s: %w", a, p, err)
+ }
+ transport, named := peer.SplitAddr(ma)
+ if named != "" && named != p {
+ continue
+ }
+ if transport != nil {
+ info.Addrs = append(info.Addrs, transport)
+ }
+ }
+ return info, nil
+}
+
+// rememberAddrs keeps what a lookup learned for a dial that follows.
+func rememberAddrs(ps peerstore.Peerstore, infos ...peer.AddrInfo) {
+ for _, info := range infos {
+ if len(info.Addrs) > 0 {
+ ps.AddAddrs(info.ID, info.Addrs, peerstore.TempAddrTTL)
+ }
+ }
+}
diff --git a/internal/node/directory_client_test.go b/internal/node/directory_client_test.go
new file mode 100644
index 00000000..751556ba
--- /dev/null
+++ b/internal/node/directory_client_test.go
@@ -0,0 +1,231 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/libp2p/go-libp2p"
+ "github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/core/routing"
+ "github.com/libp2p/go-msgio"
+ "github.com/multiformats/go-multiaddr"
+ "google.golang.org/protobuf/proto"
+
+ "github.com/google/agentmesh/api"
+)
+
+// fakeDirectoryServer answers /mesh/directory on a test router's host the
+// way a router does: records by key, a peer from the router's own peerstore
+// or its peer-key record. It fails every request while failing is set.
+type fakeDirectoryServer struct {
+ h host.Host
+ mu sync.Mutex
+ records map[string]map[peer.ID][]multiaddr.Multiaddr
+ served int
+ failing bool
+}
+
+func serveDirectory(h host.Host) *fakeDirectoryServer {
+ d := &fakeDirectoryServer{h: h, records: map[string]map[peer.ID][]multiaddr.Multiaddr{}}
+ h.SetStreamHandler(api.DirectoryProtocolID, d.handle)
+ return d
+}
+
+func (d *fakeDirectoryServer) fail(failing bool) {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ d.failing = failing
+}
+
+func (d *fakeDirectoryServer) servedCount() int {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ return d.served
+}
+
+func (d *fakeDirectoryServer) providers(key []byte) []peer.ID {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ var out []peer.ID
+ for p := range d.records[string(key)] {
+ out = append(out, p)
+ }
+ return out
+}
+
+// add puts a record in as if another router had placed it.
+func (d *fakeDirectoryServer) add(key []byte, p peer.ID, addrs ...multiaddr.Multiaddr) {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ if d.records[string(key)] == nil {
+ d.records[string(key)] = map[peer.ID][]multiaddr.Multiaddr{}
+ }
+ d.records[string(key)][p] = addrs
+}
+
+func (d *fakeDirectoryServer) handle(s network.Stream) {
+ defer func() { _ = s.Close() }()
+ frame, err := msgio.NewVarintReaderSize(s, directoryMaxMessage).ReadMsg()
+ if err != nil {
+ return
+ }
+ var req api.DirectoryRequest
+ if err := proto.Unmarshal(frame, &req); err != nil {
+ return
+ }
+ d.mu.Lock()
+ d.served++
+ resp := &api.DirectoryResponse{}
+ switch {
+ case d.failing:
+ resp.Error = "the fake router is failing"
+ case req.GetProvide() != nil:
+ q := req.GetProvide()
+ var addrs []multiaddr.Multiaddr
+ for _, a := range q.GetAddrs() {
+ if ma, err := multiaddr.NewMultiaddr(a); err == nil {
+ if transport, _ := peer.SplitAddr(ma); transport != nil {
+ addrs = append(addrs, transport)
+ }
+ }
+ }
+ if d.records[string(q.GetKey())] == nil {
+ d.records[string(q.GetKey())] = map[peer.ID][]multiaddr.Multiaddr{}
+ }
+ d.records[string(q.GetKey())][s.Conn().RemotePeer()] = addrs
+ case req.GetFindProviders() != nil:
+ for p, addrs := range d.records[string(req.GetFindProviders().GetKey())] {
+ resp.Providers = append(resp.Providers, recordOf(p, addrs))
+ }
+ case req.GetFindPeer() != nil:
+ p, err := peer.Decode(req.GetFindPeer().GetPeerId())
+ if err != nil {
+ resp.Error = err.Error()
+ break
+ }
+ if addrs := d.h.Peerstore().Addrs(p); len(addrs) > 0 {
+ resp.Peer = recordOf(p, addrs)
+ } else if addrs, ok := d.records[string(api.PeerKey(p))][p]; ok {
+ resp.Peer = recordOf(p, addrs)
+ }
+ }
+ d.mu.Unlock()
+ data, _ := proto.Marshal(resp)
+ _ = msgio.NewVarintWriter(s).WriteMsg(data)
+}
+
+func recordOf(p peer.ID, addrs []multiaddr.Multiaddr) *api.PeerRecord {
+ rec := &api.PeerRecord{PeerId: p.String()}
+ for _, a := range addrs {
+ rec.Addrs = append(rec.Addrs, a.Encapsulate(multiaddr.StringCast("/p2p/"+p.String())).String())
+ }
+ return rec
+}
+
+// newDirectoryClient is a member host with a directory over the given
+// router hosts, no handshake in between: the fake routers do not gate.
+func newDirectoryClient(t *testing.T, routers ...host.Host) (host.Host, *routerDirectory) {
+ t.Helper()
+ h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = h.Close() })
+ ids := make([]peer.ID, 0, len(routers))
+ for _, r := range routers {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ if err := h.Connect(ctx, peer.AddrInfo{ID: r.ID(), Addrs: r.Addrs()}); err != nil {
+ cancel()
+ t.Fatal(err)
+ }
+ cancel()
+ ids = append(ids, r.ID())
+ }
+ d := &routerDirectory{host: h, routers: func() []peer.ID { return ids }, addrs: h.Addrs}
+ return h, d
+}
+
+// A member announces what it serves and where it is reached to a router it
+// is attached to, and looks peers and services up there. A router that
+// fails is skipped for the next one the member holds; a lookup nobody has
+// a record for is an empty answer, and a peer nobody knows is not found.
+func TestRouterDirectoryAsksTheAttachedRouters(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ r1 := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ r2 := newFakeRouterWithKey(t, r1.cpPub, r1.cpPriv, "/ip4/127.0.0.1/tcp/0")
+ dir1, dir2 := serveDirectory(r1.h), serveDirectory(r2.h)
+ member, directory := newDirectoryClient(t, r1.h, r2.h)
+ key, err := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "calculator")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := directory.Provide(ctx, key); err != nil {
+ t.Fatalf("Provide: %v", err)
+ }
+ if got := dir1.providers(key); len(got) != 1 || got[0] != member.ID() {
+ t.Fatalf("the first router holds %v, want the member's record", got)
+ }
+ if got := dir2.providers(key); len(got) != 0 {
+ t.Fatalf("the second router holds %v; a record goes to one router, which places it", got)
+ }
+
+ found, err := directory.FindProviders(ctx, key, 0)
+ if err != nil {
+ t.Fatalf("FindProviders: %v", err)
+ }
+ if len(found) != 1 || found[0].ID != member.ID() || len(found[0].Addrs) == 0 {
+ t.Fatalf("FindProviders = %v, want the member with its addresses", found)
+ }
+ if _, named := peer.SplitAddr(found[0].Addrs[0].Encapsulate(multiaddr.StringCast("/p2p/" + member.ID().String()))); named != member.ID() {
+ t.Fatalf("addresses come back without the peer suffix: %v", found[0].Addrs)
+ }
+
+ unknown, _ := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "nobody")
+ if found, err := directory.FindProviders(ctx, unknown, 0); err != nil || len(found) != 0 {
+ t.Fatalf("FindProviders of a key nobody announced = %v, %v; want none and no error", found, err)
+ }
+
+ stranger := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0").h.ID()
+ if _, err := directory.FindPeer(ctx, stranger); err != routing.ErrNotFound {
+ t.Fatalf("FindPeer of a peer nobody knows: err = %v, want %v", err, routing.ErrNotFound)
+ }
+ info, err := directory.FindPeer(ctx, member.ID())
+ if err != nil || info.ID != member.ID() || len(info.Addrs) == 0 {
+ t.Fatalf("FindPeer of the member = %v, %v; want it from the router's own connection", info, err)
+ }
+
+ // The first router fails: the second answers, and the member is none the wiser.
+ dir1.fail(true)
+ dir2.add(key, member.ID(), member.Addrs()...)
+ before := dir2.servedCount()
+ if found, err := directory.FindProviders(ctx, key, 0); err != nil || len(found) != 1 {
+ t.Fatalf("FindProviders with the first router failing = %v, %v; want the second router's answer", found, err)
+ }
+ if got := dir2.servedCount(); got != before+1 {
+ t.Fatalf("the second router served %d requests, want %d: it must be asked when the first fails", got, before+1)
+ }
+ dir2.fail(true)
+ if _, err := directory.FindProviders(ctx, key, 0); err == nil {
+ t.Fatal("FindProviders succeeded with every router failing")
+ }
+}
diff --git a/internal/node/directory_test.go b/internal/node/directory_test.go
deleted file mode 100644
index 01dc190f..00000000
--- a/internal/node/directory_test.go
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package node
-
-import (
- "context"
- "testing"
- "time"
-
- "github.com/libp2p/go-libp2p/core/peer"
-)
-
-// TestConnectingToAnUnattachedRouterHandshakes: a connection the node opens
-// to a catalog router it is not attached to, as its DHT does for a lookup,
-// is handshaken once and counted as a directory session; the attached
-// router's handshake is not repeated by it.
-func TestConnectingToAnUnattachedRouterHandshakes(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
- defer cancel()
-
- r1 := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
- r2 := newFakeRouterWithKey(t, r1.cpPub, r1.cpPriv, "/ip4/127.0.0.1/tcp/0")
- node := r1.startNode(t, ctx, append(r1.routerAddrs, r2.routerAddrs...), func(o *Options) { o.Routers = 1 })
-
- attached := node.attachedRouters()
- if len(attached) != 1 {
- t.Fatalf("attached routers = %v, want one", attached)
- }
- other := r2
- if attached[0] == r2.h.ID() {
- other = r1
- }
- if got := node.DirectoryRouters(); got != 0 {
- t.Fatalf("directory routers before any lookup = %d, want 0", got)
- }
-
- // What the DHT client does when a key lands on the other router.
- if err := node.Host.Connect(ctx, peer.AddrInfo{ID: other.h.ID(), Addrs: other.h.Addrs()}); err != nil {
- t.Fatalf("Connect: %v", err)
- }
- deadline := time.Now().Add(5 * time.Second)
- for !node.isAuthenticatedAndConnected(other.h.ID()) {
- if time.Now().After(deadline) {
- t.Fatalf("node did not authenticate to %s after connecting to it; handshakes seen there: %d", other.h.ID(), other.handshakes.Load())
- }
- time.Sleep(20 * time.Millisecond)
- }
- if got := other.handshakes.Load(); got != 1 {
- t.Errorf("unattached router saw %d handshakes, want 1", got)
- }
- if got := node.DirectoryRouters(); got != 1 {
- t.Errorf("directory routers = %d, want 1", got)
- }
- if got := node.AttachedRouters(); got != 1 {
- t.Errorf("attached routers after the directory handshake = %d, want 1", got)
- }
- if r1.handshakes.Load()+r2.handshakes.Load() != 2 {
- t.Errorf("handshakes in all = %d, want 2: one per router", r1.handshakes.Load()+r2.handshakes.Load())
- }
-}
diff --git a/internal/node/mcp.go b/internal/node/mcp.go
index 70d83c52..21435a6a 100644
--- a/internal/node/mcp.go
+++ b/internal/node/mcp.go
@@ -434,8 +434,8 @@ func (n *AgentMeshNode) fetchRemoteServiceCatalog(ctx context.Context, peerID pe
// and ensures relay circuits are available. This prevents dial backoff errors when the
// relay is behind a load-balanced DNS address or when pods advertise internal IPs.
func (n *AgentMeshNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID) {
- if n.Host == nil || n.DHT == nil {
- logger.Debugf("[Discovery] Host or DHT is nil")
+ if n.Host == nil || n.Directory == nil {
+ logger.Debugf("[Discovery] Host or directory is nil")
return
}
logger.Debugf("[Discovery] connectedness for %s: %s (conns: %d)", targetPeer, n.Host.Network().Connectedness(targetPeer), len(n.Host.Network().ConnsToPeer(targetPeer)))
@@ -467,28 +467,21 @@ func (n *AgentMeshNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID
changed := false
addrs := n.Host.Peerstore().Addrs(targetPeer)
if len(addrs) == 0 {
- logger.Debugf("[Discovery] No addresses in peerstore for %s, querying DHT...", targetPeer)
+ logger.Debugf("[Discovery] No addresses in peerstore for %s, asking the directory", targetPeer)
findCtx, cancel := context.WithTimeout(ctx, dhtLookupTimeout)
- addrInfo, err := n.DHT.FindPeer(findCtx, targetPeer)
+ addrInfo, err := n.Directory.FindPeer(findCtx, targetPeer)
cancel()
addrs = addrInfo.Addrs
- if err != nil {
- // kad-dht answers "not found" for a peer it could not dial during
- // the lookup, which a peer reached only through a router this node
- // has not handshaken with is; what the lookup learned is in the
- // peerstore all the same.
- addrs = n.Host.Peerstore().Addrs(targetPeer)
- }
switch {
case len(addrs) > 0:
changed = true
peerLookupsTotal.WithLabelValues(lookupFound).Inc()
case err != nil && !errors.Is(err, routing.ErrNotFound):
peerLookupsTotal.WithLabelValues(lookupFailed).Inc()
- logger.Warnf("[Discovery] DHT lookup for %s failed (routing table holds %d peers): %v", targetPeer, n.DHT.RoutingTable().Size(), err)
+ logger.Warnf("[Discovery] Directory lookup for %s failed: %v", targetPeer, err)
default:
peerLookupsTotal.WithLabelValues(lookupNotFound).Inc()
- logger.Infof("[Discovery] DHT lookup found no address for %s (routing table holds %d peers); trying circuits through this node's routers", targetPeer, n.DHT.RoutingTable().Size())
+ logger.Infof("[Discovery] The directory knows no address for %s; trying circuits through this node's routers", targetPeer)
}
}
logger.Debugf("[Discovery] preparePeerAddrs for %s: found %d addrs in peerstore: %v", targetPeer, len(addrs), addrs)
@@ -543,14 +536,14 @@ func (n *AgentMeshNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID
}
if len(n.Host.Peerstore().Addrs(relayID)) == 0 {
- logger.Debugf("[Discovery] No addresses for relay %s, resolving via DHT...", relayID)
+ logger.Debugf("[Discovery] No addresses for relay %s, asking the directory", relayID)
findCtx, cancel := context.WithTimeout(ctx, dhtLookupTimeout)
- addrInfo, err := n.DHT.FindPeer(findCtx, relayID)
+ addrInfo, err := n.Directory.FindPeer(findCtx, relayID)
cancel()
if err != nil {
- logger.Debugf("[Discovery] Failed to find relay %s on DHT: %v", relayID, err)
+ logger.Debugf("[Discovery] Relay %s not found in the directory: %v", relayID, err)
} else {
- n.Host.Peerstore().AddAddrs(relayID, addrInfo.Addrs, peerstore.TempAddrTTL)
+ rememberAddrs(n.Host.Peerstore(), addrInfo)
}
}
diff --git a/internal/node/mcp_handlers.go b/internal/node/mcp_handlers.go
index 0214b7af..aee8ce6b 100644
--- a/internal/node/mcp_handlers.go
+++ b/internal/node/mcp_handlers.go
@@ -144,9 +144,8 @@ func (n *AgentMeshNode) handleGetMeshInfoRemote(ctx context.Context, req *mcp.Ca
return nil, nil, err
}
responseBytes, err := json.Marshal(struct {
- PeerID string `json:"peer_id"`
- DHTSize int `json:"dht_size"`
- }{PeerID: full.PeerID, DHTSize: full.DHTSize})
+ PeerID string `json:"peer_id"`
+ }{PeerID: full.PeerID})
if err != nil {
return nil, nil, err
}
diff --git a/internal/node/mcp_handlers_additional_test.go b/internal/node/mcp_handlers_additional_test.go
index c9b919e6..0bf12edd 100644
--- a/internal/node/mcp_handlers_additional_test.go
+++ b/internal/node/mcp_handlers_additional_test.go
@@ -76,8 +76,8 @@ func TestHandleGetMeshInfo(t *testing.T) {
if _, ok := info["connected_peers"]; !ok {
t.Errorf("missing connected_peers in mesh info")
}
- if _, ok := info["dht_size"]; !ok {
- t.Errorf("missing dht_size in mesh info")
+ if _, ok := info["attached_routers"]; !ok {
+ t.Errorf("missing attached_routers in mesh info")
}
}
diff --git a/internal/node/mcp_test.go b/internal/node/mcp_test.go
index 70de3daf..ca2f04f8 100644
--- a/internal/node/mcp_test.go
+++ b/internal/node/mcp_test.go
@@ -25,7 +25,6 @@ import (
"time"
"github.com/google/agentmesh/api"
- dht "github.com/libp2p/go-libp2p-kad-dht"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/core/peerstore"
mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
@@ -88,16 +87,10 @@ func TestResolveRelayAddresses(t *testing.T) {
t.Fatal(err)
}
- // Create a real DHT bound to the mock host
- kdht, err := dht.New(localHost, dht.Mode(dht.ModeServer))
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = kdht.Close() }()
-
+ // The directory is not asked: the addresses are in the peerstore.
node := &AgentMeshNode{
Host: localHost,
- DHT: kdht,
+ Directory: &routerDirectory{host: localHost, routers: func() []peer.ID { return nil }, addrs: localHost.Addrs},
BiscuitTimeout: 500 * time.Millisecond,
}
@@ -113,20 +106,9 @@ func TestResolveRelayAddresses(t *testing.T) {
localHost.Peerstore().AddAddr(relayID, relayDirectAddr, peerstore.PermanentAddrTTL)
relayHost.Peerstore().AddAddr(relayID, relayDirectAddr, peerstore.PermanentAddrTTL)
- // Create DHT for relay
- relayDHT, err := dht.New(relayHost, dht.Mode(dht.ModeServer))
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = relayDHT.Close() }()
-
_ = mn.LinkAll()
_ = mn.ConnectAllButSelf()
- // Wait for DHT
- _, _ = kdht.RoutingTable().TryAddPeer(relayID, true, true)
- _, _ = relayDHT.RoutingTable().TryAddPeer(localHost.ID(), true, true)
-
// Create a target node behind the relay
targetHost, err := mn.GenPeer()
if err != nil {
@@ -171,11 +153,6 @@ func TestPreparePeerAddrsFallsBackToAuthenticatedRouters(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- kdht, err := dht.New(localHost, dht.Mode(dht.ModeClient))
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = kdht.Close() }()
admitted, err := mn.GenPeer()
if err != nil {
t.Fatal(err)
@@ -195,7 +172,7 @@ func TestPreparePeerAddrsFallsBackToAuthenticatedRouters(t *testing.T) {
node := &AgentMeshNode{
Host: localHost,
- DHT: kdht,
+ Directory: &routerDirectory{host: localHost, routers: func() []peer.ID { return nil }, addrs: localHost.Addrs},
BiscuitTimeout: 500 * time.Millisecond,
authenticatedRouters: map[peer.ID]bool{admitted.ID(): true, stale.ID(): true},
}
diff --git a/internal/node/node.go b/internal/node/node.go
index 5125d159..a5b5604a 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -45,11 +45,9 @@ import (
"github.com/google/agentmesh/internal/p2plimits"
"github.com/google/agentmesh/internal/ratelimit"
lru "github.com/hashicorp/golang-lru/v2"
- "github.com/ipfs/go-cid"
golog "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p"
gostream "github.com/libp2p/go-libp2p-gostream"
- dht "github.com/libp2p/go-libp2p-kad-dht"
pubsub "github.com/libp2p/go-libp2p-pubsub"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/event"
@@ -149,7 +147,7 @@ func (n *AgentMeshNode) isAdmitted(p peer.ID) bool {
type AgentMeshNode struct {
config Options
Host host.Host
- DHT *dht.IpfsDHT
+ Directory Directory
PubSub *pubsub.PubSub
Discovery *samdiscovery.Discovery
Store *Store
@@ -648,9 +646,6 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
}
logger.Debugf("[Discovery] Peer %s connected via relay/public IP (%s), marking private IP as failed", remotePeer, remoteAddr)
}
- if c.Stat().Direction == network.DirOutbound {
- n.authDirectoryRouter(ctx, remotePeer)
- }
},
// The router drops us from its authenticated set the moment our last
// connection to it closes, so a stale entry here would report a
@@ -688,32 +683,21 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
}
}
- // The DHT and gossip are served by, and serve, authenticated peers only.
+ // Gossip and the directory are served by, and serve, authenticated
+ // peers only.
gated := &meshHost{Host: h, n: n}
- // Initialize Rendezvous (DHT Client)
- kdht, err := dht.New(gated, dhtOptions(n.directorySettings())...)
- if err != nil {
- return err
- }
- n.DHT = kdht
+ // The directory: the routers this node is attached to answer for it.
+ n.Directory = &routerDirectory{host: gated, routers: n.attachedRouters, addrs: h.Addrs}
- n.services = NewServiceRegistry(n.DHT, n.config.BackendProbeTimeout)
+ n.services = NewServiceRegistry(n.Directory, n.config.BackendProbeTimeout)
n.services.reprovideNow = n.triggerReprovide
n.applyPendingEgress(ctx)
- authenticated, err := n.admitByRouters(ctx)
- if err != nil {
+ if _, err := n.admitByRouters(ctx); err != nil {
return err
}
- if authenticated {
- logger.Infof("[DHT] Bootstrapping DHT with connected router...")
- if err := n.DHT.Bootstrap(ctx); err != nil {
- logger.Warnf("[DHT] Failed to trigger DHT bootstrap: %v", err)
- }
- }
-
// Initialize Gossipsub for control plane events. StrictSign is the
// library default; it is pinned because the event validator and the
// per-author rate limit key on msg.GetFrom(), which only the signature
@@ -964,11 +948,7 @@ func (n *AgentMeshNode) startConnectionMonitor(ctx context.Context, bootstrapDur
}
if reconnected {
- logger.Infof("[Monitor] Reconnected successfully. Bootstrapping DHT...")
- if err := n.DHT.Bootstrap(ctx); err != nil {
- logger.Warnf("[DHT] Failed to trigger DHT bootstrap on reconnect: %v", err)
- }
- logger.Infof("[Monitor] Reproviding services to DHT...")
+ logger.Infof("[Monitor] Reconnected successfully. Announcing services again...")
n.services.ReprovideAll(ctx)
consecutiveFailures = 0
continue
@@ -985,24 +965,23 @@ func (n *AgentMeshNode) startConnectionMonitor(ctx context.Context, bootstrapDur
}
func (n *AgentMeshNode) RegisterStaticServices(ctx context.Context, services []api.ServiceConfig) error {
- // Wait for node to be connected to a router or DHT to be ready
- // This avoids failure if we try to register immediately after enrollment
- // before the connection is established.
+ // A router has to hold the node before a service can be announced, so
+ // a registration right after enrollment waits for the first one.
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
timeout := time.After(10 * time.Second)
-dhtLoop:
+attached:
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-timeout:
- return fmt.Errorf("timeout waiting for DHT to be ready before registering static services")
+ return fmt.Errorf("timeout waiting for a router before registering static services")
case <-ticker.C:
- if n.IsConnected() || (n.DHT != nil && n.DHT.RoutingTable().Size() > 0) {
- break dhtLoop
+ if n.IsConnected() {
+ break attached
}
}
}
@@ -2036,11 +2015,6 @@ func (n *AgentMeshNode) Teardown() error {
n.services.TeardownAll()
}
var errs []error
- if n.DHT != nil {
- if err := n.DHT.Close(); err != nil {
- errs = append(errs, err)
- }
- }
if n.Host != nil {
if err := n.Host.Close(); err != nil {
errs = append(errs, err)
@@ -2061,47 +2035,33 @@ const (
discoveryFanoutTimeout = 40 * time.Second
)
-// findProvidersByCID is the shared DHT-lookup primitive; bounds the
-// lookup so FindProvidersAsync's channel is guaranteed to close.
-func (n *AgentMeshNode) findProvidersByCID(ctx context.Context, c cid.Cid) ([]peer.AddrInfo, error) {
- if n.DHT == nil {
- return nil, fmt.Errorf("DHT not initialized")
+// findProviders asks the directory who provides key and keeps their
+// addresses for the dials that follow.
+func (n *AgentMeshNode) findProviders(ctx context.Context, key []byte) ([]peer.AddrInfo, error) {
+ if n.Directory == nil {
+ return nil, fmt.Errorf("the directory is not up")
}
lookupCtx, cancel := context.WithTimeout(ctx, dhtLookupTimeout)
defer cancel()
- // FindProvidersAsync can emit the same peer multiple times when the
- // DHT walk converges from different paths; dedupe so downstream
- // fan-out (e.g. discoverServicesByType) doesn't double-fetch.
- providersMap := make(map[peer.ID]peer.AddrInfo)
- limit := n.config.DHTLookupLimit
- if limit <= 0 {
- limit = 20
- }
- for p := range n.DHT.FindProvidersAsync(lookupCtx, c, limit) {
- providersMap[p.ID] = p
- }
- providers := make([]peer.AddrInfo, 0, len(providersMap))
- for _, p := range providersMap {
- // Same reason as the discovery loop: a revoked peer's provider
- // record outlives its ban, so without this every service lookup
- // fans out a doomed dial and returns one fewer usable provider.
+ found, err := n.Directory.FindProviders(lookupCtx, key, n.config.DHTLookupLimit)
+ if errors.Is(err, errNoDirectoryRouter) {
+ // A member not yet on the mesh knows of nobody; the lookup is
+ // counted as failed, the caller sees an empty mesh.
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ providers := make([]peer.AddrInfo, 0, len(found))
+ for _, p := range found {
+ // A revoked peer's record outlives its ban, so without this every
+ // service lookup fans out a doomed dial and returns one fewer
+ // usable provider.
if n.peerIsRevoked(p.ID) {
continue
}
- routerAddrsCount := 0
- if n.RouterPeerID != "" {
- routerAddrsCount = len(n.Host.Peerstore().Addrs(n.RouterPeerID))
- }
- logger.Infof("[Discovery] Evaluating relay for %s: RouterPeerID=%s, RouterAddrsCount=%d", p.ID, n.RouterPeerID, routerAddrsCount)
-
- for _, addr := range p.Addrs {
- logger.Infof("[Discovery] Provider %s advertised address: %s", p.ID, addr)
- }
-
- if len(p.Addrs) > 0 {
- n.Host.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.TempAddrTTL)
- }
-
+ logger.Debugf("[Discovery] Provider %s is reached at %v", p.ID, p.Addrs)
+ rememberAddrs(n.Host.Peerstore(), p)
providers = append(providers, p)
}
return providers, nil
@@ -2109,20 +2069,20 @@ func (n *AgentMeshNode) findProvidersByCID(ctx context.Context, c cid.Cid) ([]pe
// FindProvidersByName returns peers hosting a specific {type, name} service.
func (n *AgentMeshNode) FindProvidersByName(ctx context.Context, serviceType api.ServiceType, serviceName string) ([]peer.AddrInfo, error) {
- c, err := serviceNameToCID(serviceType, serviceName)
+ key, err := api.ServiceKey(serviceType, serviceName)
if err != nil {
return nil, err
}
- return n.findProvidersByCID(ctx, c)
+ return n.findProviders(ctx, key)
}
// FindProvidersByType returns peers hosting at least one service of the given type.
func (n *AgentMeshNode) FindProvidersByType(ctx context.Context, serviceType api.ServiceType) ([]peer.AddrInfo, error) {
- c, err := serviceTypeToCID(serviceType)
+ key, err := api.ServiceKey(serviceType, "")
if err != nil {
return nil, err
}
- return n.findProvidersByCID(ctx, c)
+ return n.findProviders(ctx, key)
}
// localProxyURL builds the loopback URL clients use to reach a remote service.
diff --git a/internal/node/node_metrics.go b/internal/node/node_metrics.go
index f8da526f..747ec2e4 100644
--- a/internal/node/node_metrics.go
+++ b/internal/node/node_metrics.go
@@ -101,17 +101,6 @@ var transitHandshakesTotal = promauto.NewCounterVec(
[]string{"router", "result"},
)
-// directoryHandshakesTotal counts the handshakes run with a router this node
-// connected to for its DHT without being attached to it, so the router
-// keeps the connection as a session; see authDirectoryRouter.
-var directoryHandshakesTotal = promauto.NewCounterVec(
- prometheus.CounterOpts{
- Name: "agentmesh_node_directory_handshakes_total",
- Help: "Handshakes with a router this node connected to for a DHT lookup or announcement without being attached to it, by result",
- },
- []string{"result"},
-)
-
// nodeStateCollector exports the node's own view of its mesh membership: the
// same figures GET /debug/mesh-info and /debug/token-info answer on demand,
// as gauges a scraper can keep. A node that runs unattended, such as a
@@ -123,7 +112,6 @@ type nodeStateCollector struct {
meshConnectedDesc *prometheus.Desc
connectedDesc *prometheus.Desc
authenticatedDesc *prometheus.Desc
- dhtDesc *prometheus.Desc
biscuitExpiryDesc *prometheus.Desc
servicesDesc *prometheus.Desc
routersDesc *prometheus.Desc
@@ -149,10 +137,6 @@ func newNodeStateCollector(n *AgentMeshNode) *nodeStateCollector {
"agentmesh_node_authenticated_peers",
"Peers that have completed the mesh authentication handshake with this node",
nil, nil),
- dhtDesc: prometheus.NewDesc(
- "agentmesh_node_dht_routing_table_size",
- "Peers in the Kademlia routing table",
- nil, nil),
biscuitExpiryDesc: prometheus.NewDesc(
"agentmesh_node_biscuit_expiry_timestamp_seconds",
"Unix time the node's mesh credential expires",
@@ -175,7 +159,7 @@ func newNodeStateCollector(n *AgentMeshNode) *nodeStateCollector {
func (c *nodeStateCollector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.readyDesc, c.meshConnectedDesc, c.connectedDesc, c.authenticatedDesc,
- c.dhtDesc, c.biscuitExpiryDesc, c.servicesDesc, c.routersDesc, c.routerCandsDesc, connsPerSourceIPDesc,
+ c.biscuitExpiryDesc, c.servicesDesc, c.routersDesc, c.routerCandsDesc, connsPerSourceIPDesc,
} {
ch <- d
}
@@ -199,7 +183,6 @@ func (c *nodeStateCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.AttachedRouters()), "attached")
ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.wantRouters()), "wanted")
- ch <- prometheus.MustNewConstMetric(c.routersDesc, prometheus.GaugeValue, float64(n.DirectoryRouters()), "directory")
cands := n.routerCandidates()
_, matching := selectRouters(cands, 0, n.config.RouterSelector, nil, nil)
ch <- prometheus.MustNewConstMetric(c.routerCandsDesc, prometheus.GaugeValue, float64(len(cands)), "any")
@@ -211,7 +194,6 @@ func (c *nodeStateCollector) Collect(ch chan<- prometheus.Metric) {
return true
})
ch <- prometheus.MustNewConstMetric(c.authenticatedDesc, prometheus.GaugeValue, float64(authenticated))
- ch <- prometheus.MustNewConstMetric(c.dhtDesc, prometheus.GaugeValue, float64(n.DHT.RoutingTable().Size()))
if exp, err := n.Store.LoadIdentityExpiration(); err == nil && exp > 0 {
ch <- prometheus.MustNewConstMetric(c.biscuitExpiryDesc, prometheus.GaugeValue, float64(exp))
diff --git a/internal/node/node_metrics_test.go b/internal/node/node_metrics_test.go
index 022ca533..cccd9d55 100644
--- a/internal/node/node_metrics_test.go
+++ b/internal/node/node_metrics_test.go
@@ -23,7 +23,6 @@ import (
"time"
"github.com/libp2p/go-libp2p"
- dht "github.com/libp2p/go-libp2p-kad-dht"
"github.com/libp2p/go-libp2p/core/peer"
)
@@ -80,18 +79,12 @@ func TestNodeMetricsServer(t *testing.T) {
t.Fatal(err)
}
defer func() { _ = host.Close() }()
- kad, err := dht.New(host, dht.Mode(dht.ModeServer))
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = kad.Close() }()
-
expiry := time.Now().Add(time.Hour).Unix()
if err := store.SaveIdentityExpiration(expiry); err != nil {
t.Fatal(err)
}
node.Host = host
- node.DHT = kad
+ node.Directory = &routerDirectory{host: host, routers: node.attachedRouters, addrs: host.Addrs}
node.authPeers.Store(peer.ID("peer-a"), time.Now().Add(time.Hour))
node.authPeers.Store(peer.ID("peer-b"), time.Now().Add(time.Hour))
@@ -105,7 +98,6 @@ func TestNodeMetricsServer(t *testing.T) {
"agentmesh_node_mesh_connected 0",
"agentmesh_node_connected_peers 0",
"agentmesh_node_authenticated_peers 2",
- "agentmesh_node_dht_routing_table_size 0",
"agentmesh_node_conns_per_source_ip_limit 100",
`agentmesh_node_inbound_connections_refused_total{reason="per_ip_limit"} 0`,
"agentmesh_node_biscuit_expiry_timestamp_seconds " + strconv.FormatFloat(float64(expiry), 'g', -1, 64),
diff --git a/internal/node/node_test.go b/internal/node/node_test.go
index 7606e3ce..209c961f 100644
--- a/internal/node/node_test.go
+++ b/internal/node/node_test.go
@@ -29,9 +29,7 @@ import (
"github.com/google/agentmesh/internal/ratelimit"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/libp2p/go-libp2p"
- dht "github.com/libp2p/go-libp2p-kad-dht"
"github.com/libp2p/go-libp2p/core/crypto"
- "github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-msgio"
@@ -808,63 +806,32 @@ func TestNewAgentMeshNode_DHTOptions(t *testing.T) {
}
}
-// newDHTHost returns a host wired to a DHT on the /mesh prefix. Client mode
-// keeps the host out of its peers' routing tables, so nothing dials it except
-// the code under test.
-func newDHTHost(t *testing.T, mode dht.ModeOpt) (host.Host, *dht.IpfsDHT) {
- t.Helper()
- h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
- if err != nil {
- t.Fatal(err)
- }
- t.Cleanup(func() { _ = h.Close() })
- d, err := dht.New(h, dht.Mode(mode), dht.ProtocolPrefix("/mesh"))
- if err != nil {
- t.Fatal(err)
- }
- t.Cleanup(func() { _ = d.Close() })
- return h, d
-}
-
-// waitForRoutingTable polls until the DHT has learned about a peer.
-func waitForRoutingTable(t *testing.T, d *dht.IpfsDHT) {
- t.Helper()
- deadline := time.Now().Add(5 * time.Second)
- for d.RoutingTable().Size() == 0 {
- if time.Now().After(deadline) {
- t.Fatalf("routing table still empty after 5s")
- }
- time.Sleep(20 * time.Millisecond)
- }
-}
-
func TestFindProvidersSkipsRevoked(t *testing.T) {
ctx := context.Background()
- hostA, dhtA := newDHTHost(t, dht.ModeServer)
- hostB, dhtB := newDHTHost(t, dht.ModeServer)
- if err := hostA.Connect(ctx, peer.AddrInfo{ID: hostB.ID(), Addrs: hostB.Addrs()}); err != nil {
+ router := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ dir := serveDirectory(router.h)
+ hostA, directory := newDirectoryClient(t, router.h)
+ hostB, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
t.Fatal(err)
}
- waitForRoutingTable(t, dhtA)
- waitForRoutingTable(t, dhtB)
+ t.Cleanup(func() { _ = hostB.Close() })
- c, err := serviceNameToCID(api.ServiceType_SERVICE_TYPE_MCP, "revoked-svc")
+ key, err := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "revoked-svc")
if err != nil {
t.Fatal(err)
}
- if err := dhtB.Provide(ctx, c, true); err != nil {
- t.Fatal(err)
- }
+ dir.add(key, hostB.ID(), hostB.Addrs()...)
revokedCache, err := lru.New[string, int64](10)
if err != nil {
t.Fatal(err)
}
- node := &AgentMeshNode{Host: hostA, DHT: dhtA, revokedPeers: revokedCache}
+ node := &AgentMeshNode{Host: hostA, Directory: directory, revokedPeers: revokedCache}
- // Positive control: the provider record is findable before the ban, so a
- // later empty result is the guard and not a lookup that never worked.
+ // Positive control: the record is findable before the ban, so a later
+ // empty result is the guard and not a lookup that never worked.
providers, err := node.FindProvidersByName(ctx, api.ServiceType_SERVICE_TYPE_MCP, "revoked-svc")
if err != nil {
t.Fatal(err)
diff --git a/internal/node/peer_lookup_test.go b/internal/node/peer_lookup_test.go
index 59c7b360..be2ad006 100644
--- a/internal/node/peer_lookup_test.go
+++ b/internal/node/peer_lookup_test.go
@@ -20,29 +20,20 @@ import (
"testing"
"time"
- "github.com/google/agentmesh/api"
- dht "github.com/libp2p/go-libp2p-kad-dht"
"github.com/libp2p/go-libp2p/core/test"
"github.com/libp2p/go-libp2p/p2p/net/swarm"
"github.com/multiformats/go-multiaddr"
)
-// A peer reached only through a router this node has not handshaken with:
-// the lookup learns the circuit address from a router that holds it, dials
-// it to query the peer, and is refused, since that router has not admitted
-// this node; kad-dht then answers "not found" and leaves the address in
-// dial backoff. The node must take the address the lookup learned anyway,
-// handshake with the router, and clear the backoff, or the call fails
-// twice over for a peer that is one handshake away.
+// A peer reached only through a router this node does not hold: the
+// directory names the circuit through that router, and the node handshakes
+// with it before dialing, since a relay opens a circuit only for a source it
+// admitted. The address the lookup learned is kept, and nothing is left in
+// dial backoff.
func TestPreparePeerAddrsUsesWhatAFailedLookupLearned(t *testing.T) {
held := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
- // The held router is a DHT server, as a router is, so it enters the
- // node's routing table and answers its lookups.
- heldDHT, err := dht.New(held.h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix(api.DHTProtocolPrefix))
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = heldDHT.Close() }()
+ // The held router answers the node's directory requests, as a router does.
+ serveDirectory(held.h)
other := newFakeRouterWithKey(t, held.cpPub, held.cpPriv, "/ip4/127.0.0.1/tcp/0")
// The peer reserved with the other router, which recorded the circuit;
@@ -54,13 +45,6 @@ func TestPreparePeerAddrsUsesWhatAFailedLookupLearned(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
node := held.startNode(t, ctx, held.routerAddrs)
- for node.DHT.RoutingTable().Size() == 0 {
- select {
- case <-ctx.Done():
- t.Fatal("the held router never entered the node's routing table")
- case <-time.After(20 * time.Millisecond):
- }
- }
before := counterValue(t, peerLookupsTotal.WithLabelValues(lookupFound))
node.preparePeerAddrs(ctx, target)
diff --git a/internal/node/routers.go b/internal/node/routers.go
index b287c16d..7f1ba033 100644
--- a/internal/node/routers.go
+++ b/internal/node/routers.go
@@ -162,30 +162,6 @@ func (n *AgentMeshNode) setRouterCatalog(cands []routerCandidate) {
n.Host.Peerstore().AddAddrs(c.ID, c.Addrs, peerstore.PermanentAddrTTL)
}
}
- n.pruneDHTTable(cands)
-}
-
-// pruneDHTTable drops from the DHT routing table every peer the catalog
-// does not list. The routers are the only DHT servers, so the catalog is
-// the table's source of truth; kad-dht's own liveness pings are off
-// (dhtOptions), and without this a router that left would stay in the
-// table until a lookup failed against it.
-func (n *AgentMeshNode) pruneDHTTable(cands []routerCandidate) {
- if n.DHT == nil || len(cands) == 0 {
- return
- }
- listed := make(map[peer.ID]bool, len(cands))
- for _, c := range cands {
- listed[c.ID] = true
- }
- rt := n.DHT.RoutingTable()
- for _, p := range rt.ListPeers() {
- if listed[p] {
- continue
- }
- rt.RemovePeer(p)
- logger.Infof("[DHT] Dropped %s from the routing table: the control plane no longer lists it as a router", p)
- }
}
func (n *AgentMeshNode) routerCandidates() []routerCandidate {
diff --git a/internal/node/service.go b/internal/node/service.go
index 12f0914b..278a6cb0 100644
--- a/internal/node/service.go
+++ b/internal/node/service.go
@@ -25,8 +25,6 @@ import (
"strings"
"github.com/google/agentmesh/api"
- "github.com/ipfs/go-cid"
- "github.com/multiformats/go-multihash"
)
// Service is the contract the ServiceRegistry and ingress server use.
@@ -184,29 +182,3 @@ func buildRegisterRequest(sCfg api.ServiceConfig) (*api.RegisterServiceRequest,
}
return req, nil
}
-
-// serviceKeyToCID hashes "mesh:service[:part]..." into a DHT rendezvous CID.
-func serviceKeyToCID(parts ...string) (cid.Cid, error) {
- srvKey := strings.Join(append([]string{"mesh:service"}, parts...), ":")
- hash, err := multihash.Sum([]byte(srvKey), multihash.SHA2_256, -1)
- if err != nil {
- return cid.Undef, err
- }
- return cid.NewCidV1(cid.Raw, hash), nil
-}
-
-func serviceNameToCID(serviceType api.ServiceType, serviceName string) (cid.Cid, error) {
- srvTypeStr, err := api.ServiceTypeToString(serviceType)
- if err != nil {
- return cid.Undef, err
- }
- return serviceKeyToCID(srvTypeStr, serviceName)
-}
-
-func serviceTypeToCID(serviceType api.ServiceType) (cid.Cid, error) {
- srvTypeStr, err := api.ServiceTypeToString(serviceType)
- if err != nil {
- return cid.Undef, err
- }
- return serviceKeyToCID(srvTypeStr)
-}
diff --git a/internal/node/service_registry.go b/internal/node/service_registry.go
index 360ad97d..4bccb4f2 100644
--- a/internal/node/service_registry.go
+++ b/internal/node/service_registry.go
@@ -22,13 +22,12 @@ import (
"time"
"github.com/google/agentmesh/api"
- "github.com/ipfs/go-cid"
)
-// dhtProvider is the narrow DHT surface ServiceRegistry depends on.
-// Production wires this to *dht.IpfsDHT; tests use a fake.
-type dhtProvider interface {
- Provide(ctx context.Context, c cid.Cid, broadcast bool) error
+// directoryProvider is the one directory call ServiceRegistry makes.
+// Production wires this to the node's Directory; tests use a fake.
+type directoryProvider interface {
+ Provide(ctx context.Context, key []byte) error
}
// backendProber is implemented by services that can be asked whether their
@@ -70,7 +69,7 @@ func advertisable(ctx context.Context, svc Service, probeTimeout time.Duration)
type ServiceRegistry struct {
mu sync.RWMutex
services map[string]Service
- dht dhtProvider
+ dht directoryProvider
// reprovideNow asks the node to run a reprovide cycle now, so a service
// registered after the loop last ran does not wait a whole interval to be
@@ -88,7 +87,7 @@ type ServiceRegistry struct {
// defaultDHTProbeTimeout, so callers can pass an unset
// Options.BackendProbeTimeout straight through without an explicit
// zero-check.
-func NewServiceRegistry(d dhtProvider, backendProbeTimeout time.Duration) *ServiceRegistry {
+func NewServiceRegistry(d directoryProvider, backendProbeTimeout time.Duration) *ServiceRegistry {
if backendProbeTimeout <= 0 {
backendProbeTimeout = defaultDHTProbeTimeout
}
@@ -139,11 +138,11 @@ func (r *ServiceRegistry) Register(ctx context.Context, svc Service) error {
return fmt.Errorf("init %s: %w", info.Name, err)
}
- srvNameCID, err := serviceNameToCID(info.Type, info.Name)
+ nameKey, err := api.ServiceKey(info.Type, info.Name)
if err != nil {
return err
}
- srvTypeCID, err := serviceTypeToCID(info.Type)
+ typeKey, err := api.ServiceKey(info.Type, "")
if err != nil {
return err
}
@@ -154,14 +153,14 @@ func (r *ServiceRegistry) Register(ctx context.Context, svc Service) error {
} else {
nameCtx, nameCancel := context.WithTimeout(ctx, 5*time.Second)
defer nameCancel()
- if err := r.dht.Provide(nameCtx, srvNameCID, true); err != nil {
- logger.Warnf("[ServiceRegistry] DHT Provide (name) for %s: %v", info.Name, err)
+ if err := r.dht.Provide(nameCtx, nameKey); err != nil {
+ logger.Warnf("[ServiceRegistry] Announcing %s by name: %v", info.Name, err)
}
typeCtx, typeCancel := context.WithTimeout(ctx, 5*time.Second)
defer typeCancel()
- if err := r.dht.Provide(typeCtx, srvTypeCID, true); err != nil {
- logger.Warnf("[ServiceRegistry] DHT Provide (type) for %s: %v", info.Name, err)
+ if err := r.dht.Provide(typeCtx, typeKey); err != nil {
+ logger.Warnf("[ServiceRegistry] Announcing %s by type: %v", info.Name, err)
}
}
@@ -177,7 +176,7 @@ func (r *ServiceRegistry) Register(ctx context.Context, svc Service) error {
}
if probeErr == nil {
- logger.Infof("[ServiceRegistry] Registered %s/%s (name CID: %s, type CID: %s)", info.Type, info.Name, srvNameCID, srvTypeCID)
+ logger.Infof("[ServiceRegistry] Registered and announced %s/%s", info.Type, info.Name)
} else if r.reprovideNow != nil {
r.reprovideNow()
}
@@ -297,19 +296,20 @@ Loop:
// On shutdown every service fails this way, and saying so
// would blame backends for the node stopping.
if ctx.Err() == nil {
- logger.Warnf("[ServiceRegistry] Withholding %s/%s from the DHT: backend did not answer: %v", info.Type, info.Name, err)
+ logger.Warnf("[ServiceRegistry] Withholding %s/%s from the directory: backend did not answer: %v", info.Type, info.Name, err)
}
return
}
- if srvNameCID, err := serviceNameToCID(info.Type, info.Name); err == nil {
- nameCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
- _ = r.dht.Provide(nameCtx, srvNameCID, true)
- cancel()
- }
- if srvTypeCID, err := serviceTypeToCID(info.Type); err == nil {
- typeCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
- _ = r.dht.Provide(typeCtx, srvTypeCID, true)
+ for _, name := range []string{info.Name, ""} {
+ key, err := api.ServiceKey(info.Type, name)
+ if err != nil {
+ continue
+ }
+ provideCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ if err := r.dht.Provide(provideCtx, key); err != nil && ctx.Err() == nil {
+ logger.Warnf("[ServiceRegistry] Announcing %s/%s again: %v", info.Type, info.Name, err)
+ }
cancel()
}
}()
diff --git a/internal/node/service_registry_test.go b/internal/node/service_registry_test.go
index 463ba770..9349e74f 100644
--- a/internal/node/service_registry_test.go
+++ b/internal/node/service_registry_test.go
@@ -24,18 +24,17 @@ import (
"time"
"github.com/google/agentmesh/api"
- "github.com/ipfs/go-cid"
)
type fakeDHT struct {
- calls []cid.Cid
+ calls [][]byte
failNth int
count int32
}
-func (f *fakeDHT) Provide(ctx context.Context, c cid.Cid, _ bool) error {
+func (f *fakeDHT) Provide(ctx context.Context, key []byte) error {
n := atomic.AddInt32(&f.count, 1)
- f.calls = append(f.calls, c)
+ f.calls = append(f.calls, key)
if f.failNth > 0 && int(n) == f.failNth {
return errors.New("fake dht failure")
}
@@ -69,7 +68,7 @@ func newFakeSvc(name string, st api.ServiceType) *fakeService {
}
// newServiceRegistryForTest builds a registry against the fake DHT for tests.
-func newServiceRegistryForTest(d dhtProvider) *ServiceRegistry {
+func newServiceRegistryForTest(d directoryProvider) *ServiceRegistry {
return &ServiceRegistry{
services: map[string]Service{},
dht: d,
diff --git a/internal/node/sidecar_test.go b/internal/node/sidecar_test.go
index 3419725b..17b55e8a 100644
--- a/internal/node/sidecar_test.go
+++ b/internal/node/sidecar_test.go
@@ -28,9 +28,7 @@ import (
"github.com/google/agentmesh/api"
"github.com/libp2p/go-libp2p"
- dht "github.com/libp2p/go-libp2p-kad-dht"
- "github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
@@ -519,35 +517,17 @@ func TestRegisterService(t *testing.T) {
t.Fatal(err)
}
defer func() { _ = h.Close() }()
- d, err := dht.New(h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/mesh"))
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = d.Close() }()
-
- // Create a second host to populate routing table
- h2, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = h2.Close() }()
- d2, err := dht.New(h2, dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/mesh"))
- if err != nil {
+ // The router the node announces to.
+ router := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ serveDirectory(router.h)
+ if err := h.Connect(context.Background(), peer.AddrInfo{ID: router.h.ID(), Addrs: router.h.Addrs()}); err != nil {
t.Fatal(err)
}
- defer func() { _ = d2.Close() }()
-
- err = h.Connect(context.Background(), peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()})
- if err != nil {
- t.Fatal(err)
- }
-
- // Wait for DHT to recognize the peer
- time.Sleep(100 * time.Millisecond)
+ d := &routerDirectory{host: h, routers: func() []peer.ID { return []peer.ID{router.h.ID()} }, addrs: h.Addrs}
node := &AgentMeshNode{BiscuitTimeout: 500 * time.Millisecond,
- services: NewServiceRegistry(d, 0),
- DHT: d,
+ services: NewServiceRegistry(d, 0),
+ Directory: d,
}
// MCPService.Init() opens a live session to the URL; serve a fake MCP backend.
@@ -594,48 +574,32 @@ func TestHandleDiscoverService(t *testing.T) {
t.Fatal(err)
}
defer func() { _ = h.Close() }()
- d, err := dht.New(h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/mesh"))
- if err != nil {
+ router := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ dir := serveDirectory(router.h)
+ if err := h.Connect(context.Background(), peer.AddrInfo{ID: router.h.ID(), Addrs: router.h.Addrs()}); err != nil {
t.Fatal(err)
}
- defer func() { _ = d.Close() }()
+ d := &routerDirectory{host: h, routers: func() []peer.ID { return []peer.ID{router.h.ID()} }, addrs: h.Addrs}
node := &AgentMeshNode{BiscuitTimeout: 500 * time.Millisecond,
services: NewServiceRegistry(d, 0),
- DHT: d,
+ Directory: d,
Host: h,
BoundHTTPAddr: "127.0.0.1:8080",
}
- // Register a service on another host to be discovered
+ // A service another member announced to the directory.
h2, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
if err != nil {
t.Fatal(err)
}
defer func() { _ = h2.Close() }()
- d2, err := dht.New(h2, dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/mesh"))
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = d2.Close() }()
-
- err = h.Connect(context.Background(), peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()})
- if err != nil {
- t.Fatal(err)
- }
-
serviceInfo := &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "remote-service"}
- c, err := serviceNameToCID(serviceInfo.Type, serviceInfo.Name)
+ key, err := api.ServiceKey(serviceInfo.Type, serviceInfo.Name)
if err != nil {
t.Fatal(err)
}
- // We don't strictly need Provide to succeed if we can mock the DHT lookup,
- // but here we are using real DHT. If it fails because table is empty,
- // we might need to ensure routing table is populated.
- // Let's ignore the error for now to see if it works without it (maybe DHT cache works).
- _ = d2.Provide(context.Background(), c, true)
-
- time.Sleep(100 * time.Millisecond) // Wait for DHT
+ dir.add(key, h2.ID(), h2.Addrs()...)
req := httptest.NewRequest("GET", "/mesh/service/discover?type=mcp&name=remote-service", nil)
rr := httptest.NewRecorder()
@@ -711,70 +675,6 @@ func TestListLocalServices_TypeFilter(t *testing.T) {
}
}
-func TestServiceTypeToCID_Properties(t *testing.T) {
- mcp1, err := serviceTypeToCID(api.ServiceType_SERVICE_TYPE_MCP)
- if err != nil {
- t.Fatal(err)
- }
- mcp2, err := serviceTypeToCID(api.ServiceType_SERVICE_TYPE_MCP)
- if err != nil {
- t.Fatal(err)
- }
- if !mcp1.Equals(mcp2) {
- t.Errorf("serviceTypeToCID is non-deterministic: %s vs %s", mcp1, mcp2)
- }
-
- inf, err := serviceTypeToCID(api.ServiceType_SERVICE_TYPE_INFERENCE)
- if err != nil {
- t.Fatal(err)
- }
- if mcp1.Equals(inf) {
- t.Errorf("expected distinct CIDs for distinct types, both = %s", mcp1)
- }
-
- named, err := serviceNameToCID(api.ServiceType_SERVICE_TYPE_MCP, "some-service")
- if err != nil {
- t.Fatal(err)
- }
- if mcp1.Equals(named) {
- t.Errorf("type-only CID collided with name-keyed CID: %s", mcp1)
- }
-
- if _, err := serviceTypeToCID(api.ServiceType_SERVICE_TYPE_UNSPECIFIED); err == nil {
- t.Errorf("expected error for unspecified type")
- }
-}
-
-// TestServiceKeyToCID_Equivalence pins the wire format: the public
-// helpers must compose into the same CID as a direct call to the
-// shared key builder, so optimizing the helpers later can't silently
-// shift the DHT keys.
-func TestServiceKeyToCID_Equivalence(t *testing.T) {
- gotName, err := serviceNameToCID(api.ServiceType_SERVICE_TYPE_MCP, "svc")
- if err != nil {
- t.Fatal(err)
- }
- wantName, err := serviceKeyToCID("mcp", "svc")
- if err != nil {
- t.Fatal(err)
- }
- if !gotName.Equals(wantName) {
- t.Errorf("serviceNameToCID != serviceKeyToCID: %s vs %s", gotName, wantName)
- }
-
- gotType, err := serviceTypeToCID(api.ServiceType_SERVICE_TYPE_MCP)
- if err != nil {
- t.Fatal(err)
- }
- wantType, err := serviceKeyToCID("mcp")
- if err != nil {
- t.Fatal(err)
- }
- if !gotType.Equals(wantType) {
- t.Errorf("serviceTypeToCID != serviceKeyToCID: %s vs %s", gotType, wantType)
- }
-}
-
func TestRegisterService_Validation(t *testing.T) {
node := &AgentMeshNode{BiscuitTimeout: 500 * time.Millisecond,
services: NewServiceRegistry(&fakeDHT{}, 0),
@@ -850,60 +750,35 @@ func TestDiscoverService_Pagination(t *testing.T) {
t.Fatal(err)
}
defer func() { _ = h.Close() }()
- d, err := dht.New(h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/mesh"))
- if err != nil {
+ router := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ dir := serveDirectory(router.h)
+ if err := h.Connect(context.Background(), peer.AddrInfo{ID: router.h.ID(), Addrs: router.h.Addrs()}); err != nil {
t.Fatal(err)
}
- defer func() { _ = d.Close() }()
+ d := &routerDirectory{host: h, routers: func() []peer.ID { return []peer.ID{router.h.ID()} }, addrs: h.Addrs}
node := &AgentMeshNode{
BiscuitTimeout: 500 * time.Millisecond,
services: NewServiceRegistry(d, 0),
- DHT: d,
+ Directory: d,
Host: h,
BoundHTTPAddr: "127.0.0.1:8080",
}
- var hosts []host.Host
- var dhts []*dht.IpfsDHT
- defer func() {
- for _, hs := range hosts {
- _ = hs.Close()
- }
- for _, dt := range dhts {
- _ = dt.Close()
- }
- }()
-
serviceInfo := &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "paginated-service"}
- c, err := serviceNameToCID(serviceInfo.Type, serviceInfo.Name)
+ key, err := api.ServiceKey(serviceInfo.Type, serviceInfo.Name)
if err != nil {
t.Fatal(err)
}
-
for i := 0; i < 3; i++ {
h2, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
if err != nil {
t.Fatal(err)
}
- hosts = append(hosts, h2)
-
- d2, err := dht.New(h2, dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/mesh"))
- if err != nil {
- t.Fatal(err)
- }
- dhts = append(dhts, d2)
-
- err = h.Connect(context.Background(), peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()})
- if err != nil {
- t.Fatal(err)
- }
-
- _ = d2.Provide(context.Background(), c, true)
+ defer func() { _ = h2.Close() }()
+ dir.add(key, h2.ID(), h2.Addrs()...)
}
- time.Sleep(200 * time.Millisecond) // Wait for DHT propagation
-
// 1. Query page 1 (limit=2, offset=0)
req := httptest.NewRequest("GET", "/mesh/service/discover?type=mcp&name=paginated-service&limit=2&offset=0", nil)
rr := httptest.NewRecorder()
diff --git a/mobile/agentmesh-node-app/android/app/src/main/kotlin/dev/sammesh/connect/SamFunctions.kt b/mobile/agentmesh-node-app/android/app/src/main/kotlin/dev/sammesh/connect/SamFunctions.kt
index 4ae06119..d5c3d5e1 100644
--- a/mobile/agentmesh-node-app/android/app/src/main/kotlin/dev/sammesh/connect/SamFunctions.kt
+++ b/mobile/agentmesh-node-app/android/app/src/main/kotlin/dev/sammesh/connect/SamFunctions.kt
@@ -22,7 +22,7 @@ class SamFunctions {
/**
* Retrieves the current status and statistics of the Agent Mesh node.
- * Returns a JSON string containing connected_peers, dht_size, and node_id.
+ * Returns a JSON string containing connected_peers, attached_routers, and node_id.
*/
@AppFunction(isDescribedByKDoc = true)
suspend fun getMeshStatus(context: AppFunctionContext): String {
diff --git a/mobile/agentmesh-node-app/lib/main.dart b/mobile/agentmesh-node-app/lib/main.dart
index 240b1ecb..bcffdf9b 100644
--- a/mobile/agentmesh-node-app/lib/main.dart
+++ b/mobile/agentmesh-node-app/lib/main.dart
@@ -142,7 +142,7 @@ class _NodeControlPageState extends State<NodeControlPage> {
String _status = '';
String _nodeID = '';
int _connectedPeers = 0;
- int _dhtSize = 0;
+ int _attachedRouters = 0;
bool _loggingIn = false;
bool _devicePollingActive = false;
Timer? _pollingTimer;
@@ -976,7 +976,7 @@ class _NodeControlPageState extends State<NodeControlPage> {
if (!mounted) return;
setState(() {
_connectedPeers = info['connected_peers'] ?? 0;
- _dhtSize = info['dht_size'] ?? 0;
+ _attachedRouters = info['attached_routers'] ?? 0;
if (info['node_id'] != null) {
_nodeID = info['node_id'];
}
@@ -1106,7 +1106,7 @@ class _NodeControlPageState extends State<NodeControlPage> {
_running = false;
_status = 'Stopped';
_connectedPeers = 0;
- _dhtSize = 0;
+ _attachedRouters = 0;
}
});
}
@@ -1752,9 +1752,9 @@ class _NodeControlPageState extends State<NodeControlPage> {
const SizedBox(width: 16),
Expanded(
child: _buildStatCard(
- icon: Icons.storage,
- title: 'DHT Size',
- value: '$_dhtSize',
+ icon: Icons.hub,
+ title: 'Routers',
+ value: '$_attachedRouters',
color: Colors.purple,
),
),
diff --git a/mobile/agentmesh-node-ffi/ffi/ffi.go b/mobile/agentmesh-node-ffi/ffi/ffi.go
index af04bb5a..e1fdf41d 100644
--- a/mobile/agentmesh-node-ffi/ffi/ffi.go
+++ b/mobile/agentmesh-node-ffi/ffi/ffi.go
@@ -513,15 +513,11 @@ func GetMeshInfo() string {
}
peers := activeNode.Host.Network().Peers()
- dhtSize := 0
- if activeNode.DHT != nil && activeNode.DHT.RoutingTable() != nil {
- dhtSize = activeNode.DHT.RoutingTable().Size()
- }
resData := map[string]any{
- "connected_peers": len(peers),
- "dht_size": dhtSize,
- "node_id": activeNode.Host.ID().String(),
+ "connected_peers": len(peers),
+ "attached_routers": activeNode.AttachedRouters(),
+ "node_id": activeNode.Host.ID().String(),
}
jsonBytes, err := json.Marshal(resData)
diff --git a/tests/integration/minimal_helpers_test.go b/tests/integration/minimal_helpers_test.go
index 5f359470..7646c396 100644
--- a/tests/integration/minimal_helpers_test.go
+++ b/tests/integration/minimal_helpers_test.go
@@ -42,7 +42,7 @@ import (
"github.com/biscuit-auth/biscuit-go/v2"
"github.com/google/agentmesh/api"
"github.com/libp2p/go-libp2p"
- dht "github.com/libp2p/go-libp2p-kad-dht"
+ "github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-msgio"
@@ -278,10 +278,7 @@ func startMockRouterWithControlPlaneKey(t *testing.T) (peer.ID, string, ed25519.
_ = writer.WriteMsg(respBytes)
})
- kdht, err := dht.New(h, dht.Mode(dht.ModeServer), dht.ProtocolPrefix("/mesh"))
- if err != nil {
- t.Fatalf("failed to create DHT on mock router: %v", err)
- }
+ h.SetStreamHandler(api.DirectoryProtocolID, newMockDirectory(h).handle)
// Start HTTP server for enrollment
@@ -321,7 +318,6 @@ func startMockRouterWithControlPlaneKey(t *testing.T) (peer.ID, string, ed25519.
t.Cleanup(func() {
httpServer.Close()
- _ = kdht.Close()
_ = h.Close()
})
@@ -656,6 +652,8 @@ func extractPeerID(maddrStr string) string {
return ""
}
+// waitForDHTReady returns once the node holds a router: the directory it
+// announces to and looks peers up through.
func waitForDHTReady(t *testing.T, clientBin string, apiPort int, token string) {
t.Helper()
deadline := time.Now().Add(45 * time.Second)
@@ -669,14 +667,12 @@ func waitForDHTReady(t *testing.T, clientBin string, apiPort int, token string)
for time.Now().Before(deadline) {
cmd := exec.Command(clientBin, cmdArgs...)
out, err := cmd.Output()
- if err == nil {
- if strings.Contains(string(out), `"dht_size":`) && !strings.Contains(string(out), `"dht_size":0`) {
- return
- }
+ if err == nil && strings.Contains(string(out), `"attached_routers":["`) {
+ return
}
time.Sleep(500 * time.Millisecond)
}
- t.Fatalf("DHT not ready on port %d", apiPort)
+ t.Fatalf("node on port %d holds no router", apiPort)
}
func waitForControlPlane(t *testing.T, port int) {
@@ -846,3 +842,60 @@ func newBoundaryMCPHandler(t *testing.T) http.Handler {
})
return mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return srv }, nil)
}
+
+// mockDirectory is the directory a mock router serves: records by key, a
+// peer from the router's peerstore or its peer-key record.
+type mockDirectory struct {
+ h host.Host
+ mu sync.Mutex
+ records map[string]map[peer.ID][]string
+}
+
+func newMockDirectory(h host.Host) *mockDirectory {
+ return &mockDirectory{h: h, records: map[string]map[peer.ID][]string{}}
+}
+
+func (d *mockDirectory) handle(s network.Stream) {
+ defer func() { _ = s.Close() }()
+ reader := msgio.NewVarintReaderSize(s, 1024*64)
+ msg, err := reader.ReadMsg()
+ if err != nil {
+ return
+ }
+ var req api.DirectoryRequest
+ err = proto.Unmarshal(msg, &req)
+ reader.ReleaseMsg(msg)
+ if err != nil {
+ return
+ }
+ resp := &api.DirectoryResponse{}
+ d.mu.Lock()
+ switch {
+ case req.GetProvide() != nil:
+ q := req.GetProvide()
+ if d.records[string(q.GetKey())] == nil {
+ d.records[string(q.GetKey())] = map[peer.ID][]string{}
+ }
+ d.records[string(q.GetKey())][s.Conn().RemotePeer()] = q.GetAddrs()
+ case req.GetFindProviders() != nil:
+ for p, addrs := range d.records[string(req.GetFindProviders().GetKey())] {
+ resp.Providers = append(resp.Providers, &api.PeerRecord{PeerId: p.String(), Addrs: addrs})
+ }
+ case req.GetFindPeer() != nil:
+ if p, err := peer.Decode(req.GetFindPeer().GetPeerId()); err == nil {
+ var addrs []string
+ for _, a := range d.h.Peerstore().Addrs(p) {
+ addrs = append(addrs, a.String()+"/p2p/"+p.String())
+ }
+ if len(addrs) == 0 {
+ addrs = d.records[string(api.PeerKey(p))][p]
+ }
+ if len(addrs) > 0 {
+ resp.Peer = &api.PeerRecord{PeerId: p.String(), Addrs: addrs}
+ }
+ }
+ }
+ d.mu.Unlock()
+ respBytes, _ := proto.Marshal(resp)
+ _ = msgio.NewVarintWriter(s).WriteMsg(respBytes)
+}
diff --git a/tests/integration/openai_facade_test.go b/tests/integration/openai_facade_test.go
index 2c0e7053..9ba7a2f0 100644
--- a/tests/integration/openai_facade_test.go
+++ b/tests/integration/openai_facade_test.go
@@ -192,7 +192,8 @@ func TestOpenAIFacadeCUJ(t *testing.T) {
t.Log("OpenAI facade CUJ test passed.")
}
-// waitForDHTPeers polls get_mesh_info until the node sees DHT peers.
+// waitForDHTPeers polls get_mesh_info until the node holds a router: the
+// directory it announces to and looks peers up through.
func waitForDHTPeers(t *testing.T, apiAddr string) {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
@@ -200,13 +201,13 @@ func waitForDHTPeers(t *testing.T, apiAddr string) {
respData := callMCP(t, apiAddr, "get_mesh_info", map[string]any{})
var data map[string]any
if err := json.Unmarshal([]byte(respData), &data); err == nil {
- if dhtSize, _ := data["dht_size"].(float64); dhtSize > 0 {
+ if routers, _ := data["attached_routers"].([]any); len(routers) > 0 {
return
}
}
time.Sleep(500 * time.Millisecond)
}
- t.Fatal("DHT not ready (size 0)")
+ t.Fatal("the node holds no router")
}
// waitForFacadeModel polls the consumer's /v1/models until the model appears.
diff --git a/tests/integration/rotation_test.go b/tests/integration/rotation_test.go
index 59911ab1..9fd42c28 100644
--- a/tests/integration/rotation_test.go
+++ b/tests/integration/rotation_test.go
@@ -214,7 +214,7 @@ roles:
if err != nil {
t.Fatalf("failed to query node: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
}
- if !strings.Contains(stdout, "dht_size") {
+ if !strings.Contains(stdout, "attached_routers") {
t.Fatalf("unexpected node response: %s", stdout)
}
}
diff --git a/tests/integration/service_discovery_test.go b/tests/integration/service_discovery_test.go
index 457a04bf..22554861 100644
--- a/tests/integration/service_discovery_test.go
+++ b/tests/integration/service_discovery_test.go
@@ -73,25 +73,8 @@ func TestServiceDiscovery(t *testing.T) {
// We use the multiplexed HTTP address for MCP calls too!
connectPeer(t, actualApiAddrB, addrA)
- // Wait for DHT to have peers on Node A
- t.Log("Waiting for DHT to have peers on Node A...")
- deadline := time.Now().Add(10 * time.Second)
- var dhtReady bool
- for time.Now().Before(deadline) {
- respData := callMCP(t, actualApiAddrA, "get_mesh_info", map[string]any{})
- var data map[string]any
- if err := json.Unmarshal([]byte(respData), &data); err == nil {
- dhtSize, _ := data["dht_size"].(float64)
- if dhtSize > 0 {
- dhtReady = true
- break
- }
- }
- time.Sleep(500 * time.Millisecond)
- }
- if !dhtReady {
- t.Fatalf("DHT not ready on Node A (size 0)")
- }
+ t.Log("Waiting for Node A to hold a router...")
+ waitForDHTPeers(t, actualApiAddrA)
// Wait for DHT propagation
t.Log("Waiting for DHT propagation...")
@@ -166,25 +149,8 @@ func TestServiceDiscoveryStreaming(t *testing.T) {
// Connect Node B to Node A
connectPeer(t, actualApiAddrB, addrA)
- // Wait for DHT to have peers on Node A
- t.Log("Waiting for DHT to have peers on Node A...")
- deadline := time.Now().Add(10 * time.Second)
- var dhtReady bool
- for time.Now().Before(deadline) {
- respData := callMCP(t, actualApiAddrA, "get_mesh_info", map[string]any{})
- var data map[string]any
- if err := json.Unmarshal([]byte(respData), &data); err == nil {
- dhtSize, _ := data["dht_size"].(float64)
- if dhtSize > 0 {
- dhtReady = true
- break
- }
- }
- time.Sleep(500 * time.Millisecond)
- }
- if !dhtReady {
- t.Fatalf("DHT not ready on Node A")
- }
+ t.Log("Waiting for Node A to hold a router...")
+ waitForDHTPeers(t, actualApiAddrA)
// Wait for DHT propagation
t.Log("Waiting for DHT propagation...")