Skip to content

Agent Mesh

node, router: the mesh's replication factor is kademlia's k on a member

Antonio Ojea7c28f0ffc1ba
diff --git a/internal/node/dht.go b/internal/node/dht.go
index 71f14ade..4add2d56 100644
--- a/internal/node/dht.go
+++ b/internal/node/dht.go
@@ -38,14 +38,27 @@ import (
// and a router that leaves the catalog leaves the table (setRouterCatalog).
// Bootstrap still fills the table once at start.
//
-// The record TTL is the mesh's (ControlPlaneInfoResponse.directory): the
-// same figure the routers keep a record for, so a record the member's own
-// store holds and one a router holds expire alike.
+// The record TTL and the replication factor are the mesh's
+// (ControlPlaneInfoResponse.directory). The TTL is the same figure the
+// routers keep a record for, so a record the member's own store holds and
+// one a router holds expire alike. The replication factor r is kademlia's
+// k on the member: a record is stored on the r routers closest to its key,
+// a lookup asks those r, r at a time. The routers keep every router in
+// their own tables (router.dhtOptions) and answer FIND_NODE with the
+// closest routers to the key in order, so a member learns the r closest
+// from one answer of a router it is attached to and reaches them in one
+// hop. A member therefore holds sessions with its K attached routers plus
+// the r closest to each key it publishes or looks up, whatever the number
+// of routers; with the library's k of 20 that would be every router of a
+// mesh with fewer than 20.
func dhtOptions(directory api.DirectorySettings) []dht.Option {
return []dht.Option{
dht.Mode(dht.ModeClient),
dht.DisableAutoRefresh(),
dht.ProtocolPrefix(api.DHTProtocolPrefix),
+ dht.BucketSize(directory.Replication),
+ dht.Concurrency(directory.Replication),
+ dht.Resiliency(directory.Replication),
dht.ProviderManagerOpts(records.ProviderAddrTTL(directory.RecordTTL), records.ProvideValidity(directory.RecordTTL)),
}
}
diff --git a/internal/node/dht_test.go b/internal/node/dht_test.go
index 7cd9b359..0b3b43e8 100644
--- a/internal/node/dht_test.go
+++ b/internal/node/dht_test.go
@@ -16,6 +16,7 @@ package node
import (
"context"
+ "fmt"
"slices"
"testing"
"time"
@@ -119,3 +120,42 @@ func TestSetRouterCatalogPrunesDHTTable(t *testing.T) {
t.Fatalf("routing table after an empty catalog = %v, want [%s]", got, listed)
}
}
+
+// TestMemberDHTBucketHoldsTheReplicationFactor: the mesh's replication
+// factor is kademlia's k on the member, and a bucket of its table holds
+// that many.
+func TestMemberDHTBucketHoldsTheReplicationFactor(t *testing.T) {
+ for _, r := range []int{2, api.DefaultDHTReplication, 5} {
+ t.Run(fmt.Sprintf("replication=%d", r), func(t *testing.T) {
+ h, err := libp2p.New(libp2p.NoListenAddrs)
+ if err != nil {
+ t.Fatalf("libp2p.New: %v", err)
+ }
+ t.Cleanup(func() { _ = h.Close() })
+ settings := api.DefaultDirectorySettings()
+ settings.Replication = r
+ kdht, err := dht.New(h, dhtOptions(settings)...)
+ if err != nil {
+ t.Fatalf("dht.New: %v", err)
+ }
+ t.Cleanup(func() { _ = kdht.Close() })
+ rt := kdht.RoutingTable()
+ added := 0
+ for i := 0; i < r+1; i++ {
+ p, err := rt.GenRandPeerID(0)
+ if err != nil {
+ t.Fatalf("GenRandPeerID: %v", err)
+ }
+ // The last one is refused for capacity; any other refusal is a test bug.
+ if ok, err := rt.TryAddPeer(p, true, false); ok {
+ added++
+ } else if added < r {
+ t.Fatalf("TryAddPeer #%d refused before the bucket was full: %v", i, err)
+ }
+ }
+ if added != r {
+ t.Fatalf("a bucket took %d peers, want %d", added, r)
+ }
+ })
+ }
+}
diff --git a/internal/router/dht.go b/internal/router/dht.go
index b74e5b0e..083ef6c8 100644
--- a/internal/router/dht.go
+++ b/internal/router/dht.go
@@ -24,15 +24,27 @@ import (
"github.com/google/agentmesh/api"
)
+// dhtTableBucketSize is kademlia's k on a router: how many routers one
+// bucket of its table holds, and how many closest routers it names in a
+// FIND_NODE answer. The routers are a full mesh, so a table that holds them
+// all costs nothing, and it is what lets a member find the routers closest
+// to a key in one answer from a router it is attached to (node.dhtOptions):
+// the answer lists the closest routers in order and the member keeps the
+// first 2r. A mesh with more routers than this per bucket still works; a
+// lookup takes a hop more.
+const dhtTableBucketSize = 64
+
// dhtOptions configures the router's DHT: a server for the members and the
-// other routers. The record TTL is the mesh's (syncDirectory): three of the
-// members' reprovide intervals, so a node that is gone drops out of
-// discovery within it and two missed announcements do not unlist one that
-// is not. The library default (48h) would keep every past identity listed.
+// other routers, with every router in its table. The record TTL is the
+// mesh's (syncDirectory): three of the members' reprovide intervals, so a
+// node that is gone drops out of discovery within it and two missed
+// announcements do not unlist one that is not. The library default (48h)
+// would keep every past identity listed.
func dhtOptions(directory api.DirectorySettings) []dht.Option {
return []dht.Option{
dht.Mode(dht.ModeServer),
dht.ProtocolPrefix(api.DHTProtocolPrefix),
+ dht.BucketSize(dhtTableBucketSize),
dht.ProviderManagerOpts(records.ProviderAddrTTL(directory.RecordTTL), records.ProvideValidity(directory.RecordTTL)),
}
}
diff --git a/internal/router/dht_test.go b/internal/router/dht_test.go
new file mode 100644
index 00000000..4f51cbd1
--- /dev/null
+++ b/internal/router/dht_test.go
@@ -0,0 +1,55 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "testing"
+
+ "github.com/libp2p/go-libp2p"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+
+ "github.com/google/agentmesh/api"
+)
+
+// TestRouterDHTTableHoldsEveryRouter: a router's table bucket takes more
+// routers than the library's 20, so a member's lookup gets the closest
+// routers to its key from one answer.
+func TestRouterDHTTableHoldsEveryRouter(t *testing.T) {
+ h, err := libp2p.New(libp2p.NoListenAddrs)
+ if err != nil {
+ t.Fatalf("libp2p.New: %v", err)
+ }
+ t.Cleanup(func() { _ = h.Close() })
+ kdht, err := dht.New(h, dhtOptions(api.DefaultDirectorySettings())...)
+ if err != nil {
+ t.Fatalf("dht.New: %v", err)
+ }
+ t.Cleanup(func() { _ = kdht.Close() })
+ if got := kdht.Mode(); got != dht.ModeServer {
+ t.Fatalf("router DHT mode = %v, want %v", got, dht.ModeServer)
+ }
+
+ rt := kdht.RoutingTable()
+ const routers = 40
+ for i := 0; i < routers; i++ {
+ p, err := rt.GenRandPeerID(0)
+ if err != nil {
+ t.Fatalf("GenRandPeerID: %v", err)
+ }
+ if ok, err := rt.TryAddPeer(p, true, false); !ok {
+ t.Fatalf("router #%d refused from the table: %v; a bucket must hold at least %d", i+1, err, routers)
+ }
+ }
+}