Skip to content

Agent Mesh

node: stop refreshing the DHT table; the catalog prunes it

Antonio Ojeaf65254b77e71
diff --git a/internal/node/dht.go b/internal/node/dht.go
index 5268de8e..7b5f8c74 100644
--- a/internal/node/dht.go
+++ b/internal/node/dht.go
@@ -27,6 +27,16 @@ import (
// on a 400-connection budget, answer whoever connects, and be trusted for
// unsigned provider records. A public member is still dialled directly; the
// role in the directory and the way it is reached are separate things.
+//
+// The table is not refreshed. kad-dht treats a table of ten peers or fewer
+// as starved and, every two minutes, pings every entry and runs a lookup
+// for the member's own ID against all of them; with the routers as the only
+// servers that is a connection to every router the member is not attached
+// to, closed by the router at its auth deadline and opened again two
+// minutes later, for nothing the member needs: the servers are the routers
+// the control plane lists, learned on attach and from closer-peer answers,
+// and a router that leaves the catalog leaves the table (setRouterCatalog).
+// Bootstrap still fills the table once at start.
func dhtOptions(cfg Options) []dht.Option {
providerTTL := cfg.DHTProviderAddrTTL
if providerTTL <= 0 {
@@ -34,6 +44,7 @@ func dhtOptions(cfg Options) []dht.Option {
}
opts := []dht.Option{
dht.Mode(dht.ModeClient),
+ dht.DisableAutoRefresh(),
dht.ProtocolPrefix(api.DHTProtocolPrefix),
dht.ProviderManagerOpts(records.ProviderAddrTTL(providerTTL), records.ProvideValidity(providerTTL)),
}
diff --git a/internal/node/dht_test.go b/internal/node/dht_test.go
index 5cf36543..cfcb2ce4 100644
--- a/internal/node/dht_test.go
+++ b/internal/node/dht_test.go
@@ -23,7 +23,9 @@ import (
"github.com/libp2p/go-libp2p"
dht "github.com/libp2p/go-libp2p-kad-dht"
"github.com/libp2p/go-libp2p/core/host"
+ "github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-libp2p/core/protocol"
+ "github.com/libp2p/go-libp2p/core/test"
"github.com/google/agentmesh/api"
)
@@ -81,3 +83,39 @@ func TestMemberDHTStaysClientWhenPublic(t *testing.T) {
t.Errorf("member on a public host serves %s; protocols: %v", api.DHTProtocolID, member.Mux().Protocols())
}
}
+
+// TestSetRouterCatalogPrunesDHTTable: with kad-dht's liveness pings off, the
+// catalog is what takes a router out of the member's DHT table.
+func TestSetRouterCatalogPrunesDHTTable(t *testing.T) {
+ h, err := libp2p.New(libp2p.NoListenAddrs)
+ if err != nil {
+ t.Fatalf("libp2p.New: %v", err)
+ }
+ t.Cleanup(func() { _ = h.Close() })
+ kdht, err := dht.New(h, dhtOptions(Options{})...)
+ if err != nil {
+ t.Fatalf("dht.New: %v", err)
+ }
+ t.Cleanup(func() { _ = kdht.Close() })
+
+ listed, gone := test.RandPeerIDFatal(t), test.RandPeerIDFatal(t)
+ for _, p := range []peer.ID{listed, gone} {
+ if added, err := kdht.RoutingTable().TryAddPeer(p, false, false); err != nil || !added {
+ t.Fatalf("TryAddPeer(%s) = %v, %v; want added", p, added, err)
+ }
+ }
+
+ n := &AgentMeshNode{config: Options{Routers: 2}, Host: h, DHT: kdht}
+ n.setRouterCatalog([]routerCandidate{{AddrInfo: peer.AddrInfo{ID: listed}}})
+
+ got := kdht.RoutingTable().ListPeers()
+ if len(got) != 1 || got[0] != listed {
+ t.Fatalf("routing table after the catalog dropped %s = %v, want [%s]", gone, got, listed)
+ }
+
+ // An empty catalog says nothing about the routers and must not empty the table.
+ n.setRouterCatalog(nil)
+ if got := kdht.RoutingTable().ListPeers(); len(got) != 1 {
+ t.Fatalf("routing table after an empty catalog = %v, want [%s]", got, listed)
+ }
+}
diff --git a/internal/node/routers.go b/internal/node/routers.go
index 7f1ba033..b287c16d 100644
--- a/internal/node/routers.go
+++ b/internal/node/routers.go
@@ -162,6 +162,30 @@ func (n *AgentMeshNode) setRouterCatalog(cands []routerCandidate) {
n.Host.Peerstore().AddAddrs(c.ID, c.Addrs, peerstore.PermanentAddrTTL)
}
}
+ n.pruneDHTTable(cands)
+}
+
+// pruneDHTTable drops from the DHT routing table every peer the catalog
+// does not list. The routers are the only DHT servers, so the catalog is
+// the table's source of truth; kad-dht's own liveness pings are off
+// (dhtOptions), and without this a router that left would stay in the
+// table until a lookup failed against it.
+func (n *AgentMeshNode) pruneDHTTable(cands []routerCandidate) {
+ if n.DHT == nil || len(cands) == 0 {
+ return
+ }
+ listed := make(map[peer.ID]bool, len(cands))
+ for _, c := range cands {
+ listed[c.ID] = true
+ }
+ rt := n.DHT.RoutingTable()
+ for _, p := range rt.ListPeers() {
+ if listed[p] {
+ continue
+ }
+ rt.RemovePeer(p)
+ logger.Infof("[DHT] Dropped %s from the routing table: the control plane no longer lists it as a router", p)
+ }
}
func (n *AgentMeshNode) routerCandidates() []routerCandidate {
diff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.md
index 02b1835a..24231b48 100644
--- a/site/content/docs/guides/scalability.md
+++ b/site/content/docs/guides/scalability.md
@@ -9,25 +9,23 @@ members. It explains what each component spends per member, which setting
bounds what, how to size a deployment for the fleet you expect, how to
place members on routers with labels, and which metrics tell you that a
bound is near. The numbers come from the public testnet: six routers on
-2 vCPU held 10,000 members, each router at 8800 to 9250 connections, and
-every member stayed on the mesh for the 40 minutes the fleet was held;
-an earlier fleet of 3200 stayed through a rollout of every router and
-both control plane replicas.
+2 vCPU held 10,000 members, and every member stayed on the mesh for the
+40 minutes the fleet was held; an earlier fleet of 3200 stayed through a
+rollout of every router and both control plane replicas.
## What a member costs
A member is one `agentmesh-node` process. It holds a session with two
routers (`--routers`), chosen from those the control plane lists, and a
relay reservation on each. Through those sessions it keeps a GossipSub
-subscription for control plane events. Its DHT client also connects to
-the other routers: a router closes such a connection a minute later
-(`--auth-deadline`), since no session was opened on it, and the DHT
-client opens it again at its next lookup or at its next table refresh,
-which runs every two minutes and queries every router. A router
-therefore carries two kinds of connection: the sessions of its own
-members, which the watermarks count, and a connection from most of the
-other members most of the time. On the testnet at 10,000 members each
-router held 3200 to 5100 sessions and about 9000 connections.
+subscription for control plane events. Its DHT client connects to a
+router it is not attached to only to look a peer or a service up, or to
+re-announce a service it publishes (every 5 minutes); a router closes such a connection a minute later
+(`--auth-deadline`), since no session was opened on it, and the client
+keeps no table maintenance of its own that would open it again. A
+router therefore carries the sessions of its own members, which the
+watermarks count, plus the lookups and announcements of the last minute
+from members attached elsewhere.
A member connects to another member only to call it, through that
member's router, where it runs the credential handshake before asking for
the circuit. At rest it uses about 55 MiB of memory, 300 goroutines and
@@ -158,15 +156,15 @@ NAT entry at each end.
### What a middlebox sees per member
A member holds K long-lived connections to its routers, kept alive by
-the transport, so K entries that do not churn. Its DHT client holds a
-connection to each other router; the router closes that connection at
-`--auth-deadline` (a minute) since no session was opened on it, and the
-DHT client's next lookup or table refresh, every two minutes, opens it
-again, so a mesh of R routers adds R − K entries per member that are
-created and torn down about once a minute. At 10,000 members on six
-routers that is about 3700 new connections a minute arriving at each
-router. A member that reaches a peer directly holds one entry per such
-peer, at both ends. A relayed connection adds none.
+the transport, so K entries that do not churn. A lookup, and every
+re-announcement of a service it publishes, opens a connection to each
+other router, which the router closes at `--auth-deadline` (a minute)
+since no session was opened on it: R − K short-lived entries per lookup
+or announcement. A member that publishes a service adds them every 5 minutes; one
+that publishes
+nothing adds them only when it looks something up. A member that
+reaches a peer directly holds one entry per such peer, at both ends. A
+relayed connection adds none.
A firewall between a member and the routers needs TCP (and UDP for
QUIC, when the member is allowed to fall back to it) to the routers'
@@ -204,34 +202,31 @@ those the full routers send away; the members already placed do not move
on their own. When a router stops, its members spread over the N − 1
others, which is where the headroom in the next section goes.
-Connections are more than sessions. The members on the other routers
-keep a DHT connection to this one about four fifths of the time, so a
-router holds about F × K / N + 0.8 × F × (N − K) / N connections, which
-at six routers is 0.9 × F: adding a router divides the sessions, and
-leaves the connections where they were. What bounds the connections is
-the router's libp2p connection cap, twice the high watermark, and the
-watermarks do not move it: a router past the cap refuses the next
-connection, session or not. A fleet is therefore also bounded by
-2 × H members on any one router, whatever N.
+Connections are sessions plus what is in passing. A lookup or an
+announcement from a member attached elsewhere is a connection to this
+router for at most a minute, so a router holds about F × K / N sessions
+plus, for P providers re-announcing every I minutes, P × (N − K) / N / I
+announcement connections, plus one per router per lookup of the last
+minute. With every one of 10,000 members a provider on six routers at the
+default interval that is about 1300 on top of 3300 sessions. What bounds
+the total is the router's libp2p connection cap, twice the high
+watermark: a router past it refuses the next connection, session or not.
## Sizing the routers
Start from the fleet you expect, F members on N routers with `--routers`
at its default of 2, and size for the day one router is down:
-- `--high-watermark` at least 1.2 × F × 2 / (N − 1), and at least
- 0.55 × F so that the connection cap, twice the high mark, holds the
- sessions and the DHT connections of the whole fleet with a router out.
- Each router then holds F × 2 / N sessions in normal running, with room
- for the members of a stopped router and for the connections members
- open in passing (a call to a peer on this router, two connections each
- while they enroll). 5000 members on four routers need 4000; 10,000
- members need six routers at 6000, where the second rule is the one
- that binds. The testnets deploy with 6000 (`ROUTER_HIGH_WATERMARK` in
- the deploy workflow) on six routers (`ROUTER_REPLICAS`), which held
- 10,000 members at about 9000 connections per router; with one router
- out the five others carry about the same, since the DHT connections
- are most of the count, of a cap of 12,000.
+- `--high-watermark` at least 1.2 × F × 2 / (N − 1). Each router then
+ holds F × 2 / N sessions in normal running, with room for the members
+ of a stopped router, and the connection cap at twice the high mark
+ holds the connections members open in passing: lookups, service
+ announcements, a call to a peer on this router, two connections each
+ while they enroll. 5000 members on four routers need 4000; 10,000
+ members need six routers at 4800 or seven at 4000. The testnets deploy
+ with 6000 (`ROUTER_HIGH_WATERMARK` in the deploy workflow) on six
+ routers (`ROUTER_REPLICAS`), which holds 10,000 members with one
+ router out.
- `--low-watermark` at its default, 15% below the high mark. The gap is
what one shedding moves to the other routers; a smaller gap sheds more
often, a larger one moves more members at once. A router that sheds is
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index 46fbb663..59c6eafd 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -42,7 +42,7 @@ replica count.
| `--lease-renew-interval` | `300s` | How often the lease is renewed. Must be well below the control plane's `--lease-duration`. |
| `--allow-loopback` | `false` | Announce and accept loopback and link-local addresses. For a router and nodes on one host. |
| `--conns-per-source-ip` | a quarter of `--high-watermark` (`1000`) | Inbound connections accepted per source address. One address can hold at most this share of the router's connection budget, so filling a router takes at least four addresses. Members behind a NAT or a cluster's SNAT share one address and each holds one connection per router, two while enrolling. |
-| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | How many members the router holds: peers that passed the handshake, other routers aside. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The same numbers bound connections: above the high mark the connection manager closes connections, oldest first among those that carry no session, then members; connections to the other routers are never closed. A connection without a session lasts at most `--auth-deadline`, and a member attached elsewhere opens one to this router about once a minute for its DHT, so at scale a router holds about as many of those as it has sessions, or more; the connection manager leaves them alone while they are younger than its grace period, and libp2p's connection cap, twice the high mark, is what bounds the total (see the [scalability guide](../../guides/scalability/#what-bounds-the-mesh)). |
+| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | How many members the router holds: peers that passed the handshake, other routers aside. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The same numbers bound connections: above the high mark the connection manager closes connections, oldest first among those that carry no session, then members; connections to the other routers are never closed. A connection without a session lasts at most `--auth-deadline`; a member attached elsewhere opens one to this router when it looks a peer or service up or re-announces a service, so those come and go with the mesh's activity; the connection manager leaves them alone while they are younger than its grace period, and libp2p's connection cap, twice the high mark, is what bounds the total (see the [scalability guide](../../guides/scalability/#what-bounds-the-mesh)). |
| `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. |
| `--dht-max-record-age` | library default | DHT value record lifetime. |
| `--relay-limit-duration`, `--relay-limit-data` | `1h`, `0` | Caps on each relayed connection: lifetime, and bytes per direction (`512MiB`, `1GB`). The relay cuts the connection when either is reached. `0` means no limit. |