Skip to content

Agent Mesh

controlplane: the sweep bounds every table enrollment grows, and the catalog cache

Antonio Ojeab9ae58fab079
diff --git a/internal/controlplane/catalog.go b/internal/controlplane/catalog.go
index 820b703e..ab79b0ac 100644
--- a/internal/controlplane/catalog.go
+++ b/internal/controlplane/catalog.go
@@ -93,6 +93,7 @@ func (s *Server) dropCatalogEntry(peerID string) {
}
s.catalogMu.Lock()
delete(s.catalog, pID.String())
+ catalogEntries.Set(float64(len(s.catalog)))
s.catalogMu.Unlock()
}
@@ -144,6 +145,7 @@ func (s *Server) HandleNodeCatalog(w http.ResponseWriter, r *http.Request) {
Services: req.Services,
ReportedAt: time.Now(),
}
+ catalogEntries.Set(float64(len(s.catalog)))
s.catalogMu.Unlock()
w.WriteHeader(http.StatusNoContent)
diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.go
index fb1f2a84..29adf675 100644
--- a/internal/controlplane/metrics.go
+++ b/internal/controlplane/metrics.go
@@ -113,6 +113,10 @@ type meshSnapshot struct {
// the retiring one stops verifying; zero when there is none.
signingKeys int
retiringKeyExpiry time.Time
+ // storageRows is the row count of every table the control plane keeps
+ // bounded; a table that grows past its retention is a reaper that
+ // stopped working.
+ storageRows map[string]int64
}
// meshStateCollector exports the control plane's view of the mesh: who is
@@ -141,6 +145,7 @@ type meshStateCollector struct {
meshPeersDesc *prometheus.Desc
signingKeysDesc *prometheus.Desc
retiringKeyDesc *prometheus.Desc
+ storageRowsDesc *prometheus.Desc
scrapeOKDesc *prometheus.Desc
scrapeSampleDesc *prometheus.Desc
}
@@ -195,6 +200,10 @@ func newMeshStateCollector(store storage.Store) *meshStateCollector {
"agentmesh_control_plane_retiring_key_expiry_timestamp_seconds",
"Unix time the retiring signing key stops verifying; absent when no rotation is in its grace period",
nil, nil),
+ storageRowsDesc: prometheus.NewDesc(
+ "agentmesh_control_plane_storage_rows",
+ "Rows in a control plane store table",
+ []string{"table"}, nil),
scrapeOKDesc: prometheus.NewDesc(
"agentmesh_control_plane_mesh_state_scrape_success",
"1 if the mesh state was read from the store, 0 if the last read failed",
@@ -210,7 +219,7 @@ func (c *meshStateCollector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.nodesDesc, c.usersDesc, c.requestsDesc, c.tokensDesc, c.routersDesc,
c.routerPeersDesc, c.routerDHTDesc, c.routerLeaseDesc, c.meshPeersDesc,
- c.signingKeysDesc, c.retiringKeyDesc, c.scrapeOKDesc, c.scrapeSampleDesc,
+ c.signingKeysDesc, c.retiringKeyDesc, c.storageRowsDesc, c.scrapeOKDesc, c.scrapeSampleDesc,
} {
ch <- d
}
@@ -257,6 +266,9 @@ func (c *meshStateCollector) Collect(ch chan<- prometheus.Metric) {
if !s.retiringKeyExpiry.IsZero() {
ch <- prometheus.MustNewConstMetric(c.retiringKeyDesc, prometheus.GaugeValue, float64(s.retiringKeyExpiry.Unix()))
}
+ for table, n := range s.storageRows {
+ ch <- prometheus.MustNewConstMetric(c.storageRowsDesc, prometheus.GaugeValue, float64(n), table)
+ }
}
// refresh replaces the snapshot from the store. A failed read keeps the last
@@ -335,6 +347,11 @@ func readMeshSnapshot(ctx context.Context, store storage.Store, now time.Time) (
}
}
+ snap.storageRows, err = store.CountRows(ctx)
+ if err != nil {
+ return snap, err
+ }
+
return snap, nil
}
diff --git a/internal/controlplane/metrics_test.go b/internal/controlplane/metrics_test.go
index a7b0044c..099d97d7 100644
--- a/internal/controlplane/metrics_test.go
+++ b/internal/controlplane/metrics_test.go
@@ -343,6 +343,10 @@ func TestMetricsEndpointServesMeshStateAndRuntime(t *testing.T) {
"agentmesh_control_plane_mesh_state_scrape_success 1",
"agentmesh_control_plane_routers_active 0",
"agentmesh_control_plane_mesh_connected_peers 0",
+ `agentmesh_control_plane_storage_rows{table="nodes"} 0`,
+ `agentmesh_control_plane_storage_rows{table="enrollment_requests"} 0`,
+ "agentmesh_control_plane_catalog_entries ",
+ `agentmesh_control_plane_storage_rows_deleted_total{table="routers"} `,
"go_goroutines",
} {
if !strings.Contains(string(body), want) {
diff --git a/internal/controlplane/node_gc.go b/internal/controlplane/node_gc.go
index 83a32d9a..8056f15f 100644
--- a/internal/controlplane/node_gc.go
+++ b/internal/controlplane/node_gc.go
@@ -20,6 +20,8 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
+
+ "github.com/google/agentmesh/internal/storage"
)
const (
@@ -60,31 +62,136 @@ const (
deletedReasonEphemeral = "ephemeral"
)
+var storageRowsDeletedTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+ Name: "agentmesh_control_plane_storage_rows_deleted_total",
+ Help: "Rows the storage GC removed from a table other than nodes: enrollment requests older than the retention, bootstrap tokens expired for the retention that no request refers to, router leases expired for the retention",
+}, []string{"table"})
+
+var catalogEntries = promauto.NewGauge(prometheus.GaugeOpts{
+ Name: "agentmesh_control_plane_catalog_entries",
+ Help: "Service reports the control plane holds in memory, one per node that reported since it was admitted",
+})
+
+const (
+ tableEnrollmentRequests = "enrollment_requests"
+ tableBootstrapTokens = "bootstrap_tokens"
+ tableRouters = "routers"
+)
+
func init() {
for _, reason := range []string{deletedReasonSession, deletedReasonSilent, deletedReasonEphemeral} {
nodesDeletedTotal.WithLabelValues(reason)
}
+ for _, table := range []string{tableEnrollmentRequests, tableBootstrapTokens, tableRouters} {
+ storageRowsDeletedTotal.WithLabelValues(table)
+ }
}
// runNodeGCLoop sweeps once at start and then hourly. Replicas each sweep
-// against the shared database; the delete is idempotent, so the race is
+// against the shared database; the deletes are idempotent, so the race is
// harmless and not worth a claim.
func (s *Server) runNodeGCLoop() {
defer s.wg.Done()
- s.gcExpiredNodes(time.Now())
+ s.gcStorage(time.Now())
ticker := time.NewTicker(nodeGCInterval(s.config.NodeRetention))
defer ticker.Stop()
for {
select {
case <-ticker.C:
- s.gcExpiredNodes(time.Now())
+ s.gcStorage(time.Now())
case <-s.ctx.Done():
return
}
}
}
+// gcStorage is one sweep over everything that grows with mesh activity: the
+// node records, the enrollment requests, the bootstrap tokens, the router
+// leases, and the in-memory service catalog. Each step logs its own failure
+// and the next one still runs, so one table's trouble does not let another
+// grow.
+func (s *Server) gcStorage(now time.Time) {
+ s.gcExpiredNodes(now)
+ s.gcEnrollmentRecords(now)
+ s.pruneCatalog(now)
+}
+
+// gcEnrollmentRecords deletes the enrollment requests resolved, or left
+// pending, more than NodeRetention ago (EphemeralNodeRetention when an
+// ephemeral token made them), then the bootstrap tokens that expired more
+// than NodeRetention ago and that no remaining request refers to, then the
+// router leases expired for as long. Requests go before tokens because a
+// request refers to its token.
+func (s *Server) gcEnrollmentRecords(now time.Time) {
+ ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
+ defer cancel()
+
+ before := now.Add(-s.config.NodeRetention)
+ requests, err := s.store.DeleteStaleEnrollmentRequests(ctx, before, now.Add(-s.config.EphemeralNodeRetention))
+ if err != nil {
+ logger.Errorf("Failed to delete stale enrollment requests: %v", err)
+ } else if requests > 0 {
+ storageRowsDeletedTotal.WithLabelValues(tableEnrollmentRequests).Add(float64(requests))
+ logger.Infof("Deleted %d enrollment requests older than the retention", requests)
+ }
+
+ tokens, err := s.store.DeleteExpiredBootstrapTokens(ctx, before)
+ if err != nil {
+ logger.Errorf("Failed to delete expired bootstrap tokens: %v", err)
+ } else if tokens > 0 {
+ storageRowsDeletedTotal.WithLabelValues(tableBootstrapTokens).Add(float64(tokens))
+ logger.Infof("Deleted %d bootstrap tokens expired more than %s ago", tokens, s.config.NodeRetention)
+ }
+
+ leases, err := s.store.DeleteExpiredRouterLeases(ctx, before)
+ if err != nil {
+ logger.Errorf("Failed to delete expired router leases: %v", err)
+ } else if leases > 0 {
+ storageRowsDeletedTotal.WithLabelValues(tableRouters).Add(float64(leases))
+ logger.Infof("Deleted %d router leases expired more than %s ago", leases, s.config.NodeRetention)
+ }
+}
+
+// pruneCatalog forgets the service report of every peer that is no longer
+// an admitted node: its enrollment lapsed, was collected or was banned on
+// another replica. The console already hides such reports; this keeps the
+// cache from holding one entry per member that ever joined until the next
+// restart.
+func (s *Server) pruneCatalog(now time.Time) {
+ ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
+ defer cancel()
+
+ snap := s.catalogSnapshot()
+ if len(snap) == 0 {
+ catalogEntries.Set(0)
+ return
+ }
+ admitted, err := s.store.ListNodes(ctx, storage.NodeFilter{States: []storage.NodeState{storage.NodeAdmitted}, Now: now})
+ if err != nil {
+ logger.Errorf("Failed to list admitted nodes to prune the service catalog: %v", err)
+ return
+ }
+ keep := make(map[string]struct{}, len(admitted))
+ for i := range admitted {
+ keep[admitted[i].PeerID] = struct{}{}
+ }
+ dropped := 0
+ s.catalogMu.Lock()
+ for peerID := range snap {
+ if _, ok := keep[peerID]; !ok {
+ delete(s.catalog, peerID)
+ dropped++
+ }
+ }
+ remaining := len(s.catalog)
+ s.catalogMu.Unlock()
+ catalogEntries.Set(float64(remaining))
+ if dropped > 0 {
+ logger.Infof("Dropped %d service reports of nodes no longer admitted, %d remain", dropped, remaining)
+ }
+}
+
// gcExpiredNodes deletes every unbanned node whose session expired more than
// NodeRetention before now, and every unbanned bootstrap member, which has
// no session, that did not refresh for NodeRetention, or for
diff --git a/internal/controlplane/node_gc_test.go b/internal/controlplane/node_gc_test.go
index f504ba6f..b8573556 100644
--- a/internal/controlplane/node_gc_test.go
+++ b/internal/controlplane/node_gc_test.go
@@ -21,6 +21,8 @@ import (
"testing"
"time"
+ dto "github.com/prometheus/client_model/go"
+
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/storage"
)
@@ -139,6 +141,102 @@ func TestGCForgetsSilentBootstrapMembers(t *testing.T) {
}
}
+// One sweep also bounds the records enrollment leaves behind: the requests,
+// the tokens that made them, the router leases, and the in-memory service
+// catalog. Every table the testnet audit found growing is covered here.
+func TestGCStorageBoundsEnrollmentRecordsAndCatalog(t *testing.T) {
+ store := newMetricsTestStore(t)
+ ctx := context.Background()
+ now := time.Now()
+ day := 24 * time.Hour
+
+ if err := store.SaveBootstrapToken(ctx, &storage.BootstrapToken{
+ ID: "old", TokenHash: "hash-old", Role: api.RoleNode, MaxUsages: 10, CreatedAt: now.Add(-41 * day), ExpiresAt: now.Add(-40 * day),
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveBootstrapToken(ctx, &storage.BootstrapToken{
+ ID: "live", TokenHash: "hash-live", Role: api.RoleNode, MaxUsages: 10, CreatedAt: now, ExpiresAt: now.Add(day),
+ }); err != nil {
+ t.Fatal(err)
+ }
+ resolved := now.Add(-40 * day)
+ if err := store.CreateEnrollmentRequest(ctx, &storage.EnrollmentRequest{
+ ID: "old-request", PeerID: "old-peer", PublicKey: []byte("pub"), TokenID: "old",
+ Status: api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED, CreatedAt: now.Add(-41 * day), ResolvedAt: &resolved,
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.CreateEnrollmentRequest(ctx, &storage.EnrollmentRequest{
+ ID: "live-request", PeerID: "live-peer", PublicKey: []byte("pub"), TokenID: "live",
+ Status: api.EnrollmentStatus_ENROLLMENT_STATUS_PENDING, CreatedAt: now,
+ }); err != nil {
+ t.Fatal(err)
+ }
+ for _, lease := range []struct {
+ id string
+ expiresAt time.Time
+ }{{"router-live", now.Add(time.Minute)}, {"router-gone", now.Add(-40 * day)}} {
+ if err := store.UpsertRouterLease(ctx, &storage.RouterLease{PeerID: lease.id, Addresses: []string{"/ip4/127.0.0.1/tcp/1"}, LastRenewal: now, ExpiresAt: lease.expiresAt}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ // An admitted node that reported services, and a report left by a node
+ // whose row the sweep already collected.
+ admitted, gone := "admitted-peer", "gone-peer"
+ if err := store.EnrollNode(ctx, &storage.EnrolledNode{
+ PeerID: admitted, PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode, EnrollmentType: "BOOTSTRAP", EnrolledAt: now,
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ srv, err := NewServer(Options{
+ DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: "https://issuer",
+ NodeRetention: 30 * day, EphemeralNodeRetention: time.Hour,
+ }, store)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = srv.Close() }()
+ srv.catalog[admitted] = nodeCatalogEntry{ReportedAt: now}
+ srv.catalog[gone] = nodeCatalogEntry{ReportedAt: now}
+
+ before := map[string]float64{}
+ for _, table := range []string{tableEnrollmentRequests, tableBootstrapTokens, tableRouters} {
+ before[table] = counterValue(t, storageRowsDeletedTotal, table)
+ }
+
+ srv.gcStorage(now)
+
+ for table, want := range map[string]float64{tableEnrollmentRequests: 1, tableBootstrapTokens: 1, tableRouters: 1} {
+ if got := counterValue(t, storageRowsDeletedTotal, table) - before[table]; got != want {
+ t.Errorf("storage_rows_deleted_total{table=%s} advanced by %v, want %v", table, got, want)
+ }
+ }
+ counts, err := store.CountRows(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for table, want := range map[string]int64{"enrollment_requests": 1, "bootstrap_tokens": 1, "routers": 1, "nodes": 1} {
+ if counts[table] != want {
+ t.Errorf("%s has %d rows after the sweep, want %d", table, counts[table], want)
+ }
+ }
+ if _, ok := srv.catalogSnapshot()[gone]; ok {
+ t.Errorf("catalog still holds the report of a node that is no longer enrolled")
+ }
+ if _, ok := srv.catalogSnapshot()[admitted]; !ok {
+ t.Errorf("catalog dropped the report of an admitted node")
+ }
+ var m dto.Metric
+ if err := catalogEntries.Write(&m); err != nil {
+ t.Fatal(err)
+ }
+ if got := m.GetGauge().GetValue(); got != 1 {
+ t.Errorf("catalog_entries = %v, want 1", got)
+ }
+}
+
// A retention that is not positive is refused, not read as "keep forever":
// an unbounded nodes table is what this GC exists to prevent. Unset is the
// default (TestGCExpiredNodesDefaultsRetention).
diff --git a/internal/storage/round_trip_test.go b/internal/storage/round_trip_test.go
index 2617f021..721fd46f 100644
--- a/internal/storage/round_trip_test.go
+++ b/internal/storage/round_trip_test.go
@@ -303,6 +303,143 @@ func TestDeleteSilentBootstrapNodes(t *testing.T) {
}
}
+func TestDeleteStaleEnrollmentRecords(t *testing.T) {
+ store := newTestStore(t)
+ ctx := context.Background()
+ now := time.Now()
+ day := 24 * time.Hour
+ token := func(id string, expiresAt time.Time, ephemeral bool) {
+ t.Helper()
+ if err := store.SaveBootstrapToken(ctx, &BootstrapToken{
+ ID: id, TokenHash: "hash-" + id, Role: api.RoleNode, MaxUsages: 10,
+ CreatedAt: expiresAt.Add(-day), ExpiresAt: expiresAt, Ephemeral: ephemeral,
+ }); err != nil {
+ t.Fatalf("token %s: %v", id, err)
+ }
+ }
+ request := func(id, tokenID string, createdAt time.Time, resolvedAt *time.Time) {
+ t.Helper()
+ if err := store.CreateEnrollmentRequest(ctx, &EnrollmentRequest{
+ ID: id, PeerID: "peer-" + id, PublicKey: []byte("pub"), TokenID: tokenID,
+ Status: api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED, CreatedAt: createdAt, ResolvedAt: resolvedAt,
+ }); err != nil {
+ t.Fatalf("request %s: %v", id, err)
+ }
+ }
+ at := func(ago time.Duration) *time.Time { ts := now.Add(-ago); return &ts }
+
+ // A token still in use, one that expired long ago but whose request is
+ // recent, one expired long ago with an old request, an ephemeral one
+ // whose requests are two hours old, and one that just expired.
+ token("live", now.Add(day), false)
+ token("old-with-recent-request", now.Add(-40*day), false)
+ token("old", now.Add(-40*day), false)
+ token("ephemeral", now.Add(-2*time.Hour), true)
+ token("just-expired", now.Add(-time.Hour), false)
+ request("live-1", "live", now.Add(-time.Hour), at(time.Hour))
+ request("recent-on-old-token", "old-with-recent-request", now.Add(-41*day), at(2*day))
+ request("old-resolved", "old", now.Add(-41*day), at(40*day))
+ request("old-pending", "old", now.Add(-41*day), nil)
+ request("ephemeral-1", "ephemeral", now.Add(-2*time.Hour), at(2*time.Hour))
+ request("ephemeral-2", "ephemeral", now.Add(-2*time.Hour), at(2*time.Hour))
+ request("ephemeral-fresh", "ephemeral", now.Add(-10*time.Minute), at(10*time.Minute))
+
+ requests, err := store.DeleteStaleEnrollmentRequests(ctx, now.Add(-30*day), now.Add(-time.Hour))
+ if err != nil {
+ t.Fatalf("DeleteStaleEnrollmentRequests: %v", err)
+ }
+ if requests != 4 {
+ t.Errorf("deleted %d requests, want 4 (old-resolved, old-pending, ephemeral-1, ephemeral-2)", requests)
+ }
+ left, err := store.ListEnrollmentRequests(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var leftIDs []string
+ for _, r := range left {
+ leftIDs = append(leftIDs, r.ID)
+ }
+ slices.Sort(leftIDs)
+ if want := []string{"ephemeral-fresh", "live-1", "recent-on-old-token"}; !slices.Equal(leftIDs, want) {
+ t.Errorf("remaining requests = %v, want %v", leftIDs, want)
+ }
+
+ // "old" lost both requests and goes; "old-with-recent-request" is still
+ // referred to and stays; "ephemeral" keeps its fresh request; "just-expired"
+ // is within the retention.
+ tokens, err := store.DeleteExpiredBootstrapTokens(ctx, now.Add(-30*day))
+ if err != nil {
+ t.Fatalf("DeleteExpiredBootstrapTokens: %v", err)
+ }
+ if tokens != 1 {
+ t.Errorf("deleted %d tokens, want 1 (old)", tokens)
+ }
+ remaining, err := store.ListBootstrapTokens(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var tokenIDs []string
+ for i := range remaining {
+ tokenIDs = append(tokenIDs, remaining[i].ID)
+ }
+ slices.Sort(tokenIDs)
+ if want := []string{"ephemeral", "just-expired", "live", "old-with-recent-request"}; !slices.Equal(tokenIDs, want) {
+ t.Errorf("remaining tokens = %v, want %v", tokenIDs, want)
+ }
+}
+
+func TestDeleteExpiredRouterLeases(t *testing.T) {
+ store := newTestStore(t)
+ ctx := context.Background()
+ now := time.Now()
+ lease := func(id string, expiresAt time.Time) {
+ t.Helper()
+ if err := store.UpsertRouterLease(ctx, &RouterLease{
+ PeerID: id, Addresses: []string{"/ip4/127.0.0.1/tcp/1"}, LastRenewal: expiresAt.Add(-time.Minute), ExpiresAt: expiresAt,
+ }); err != nil {
+ t.Fatalf("lease %s: %v", id, err)
+ }
+ }
+ lease("live", now.Add(time.Minute))
+ lease("lapsed-today", now.Add(-time.Hour))
+ lease("gone", now.Add(-40*24*time.Hour))
+
+ deleted, err := store.DeleteExpiredRouterLeases(ctx, now.Add(-30*24*time.Hour))
+ if err != nil {
+ t.Fatalf("DeleteExpiredRouterLeases: %v", err)
+ }
+ if deleted != 1 {
+ t.Errorf("deleted %d leases, want 1 (gone)", deleted)
+ }
+ counts, err := store.CountRows(ctx)
+ if err != nil {
+ t.Fatalf("CountRows: %v", err)
+ }
+ if counts["routers"] != 2 {
+ t.Errorf("routers rows = %d, want 2 (live, lapsed-today); counts %v", counts["routers"], counts)
+ }
+ active, err := store.GetActiveRouters(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(active) != 1 || active[0].PeerID != "live" {
+ t.Errorf("active routers = %+v, want only live", active)
+ }
+}
+
+func TestCountRowsCoversEveryGrowingTable(t *testing.T) {
+ store := newTestStore(t)
+ counts, err := store.CountRows(context.Background())
+ if err != nil {
+ t.Fatalf("CountRows: %v", err)
+ }
+ for _, table := range []string{"nodes", "enrollment_requests", "bootstrap_tokens", "routers", "revoked_biscuits", "banned_identities", "users", "keyring"} {
+ if _, ok := counts[table]; !ok {
+ t.Errorf("CountRows lacks %q: %v", table, counts)
+ }
+ }
+}
+
func TestDeleteExpiredNodesSparesBannedAndUnbounded(t *testing.T) {
store := newTestStore(t)
ctx := context.Background()
diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.go
index 9d4ea76e..498fc4e9 100644
--- a/internal/storage/sql_store.go
+++ b/internal/storage/sql_store.go
@@ -1122,6 +1122,61 @@ func (s *SQLStore) DeleteSilentBootstrapNodes(ctx context.Context, before, ephem
return regular, ephemeral, err
}
+// DeleteStaleEnrollmentRequests implements Store.
+func (s *SQLStore) DeleteStaleEnrollmentRequests(ctx context.Context, before, ephemeralBefore time.Time) (int64, error) {
+ res, err := s.db.ExecContext(ctx, s.rebind(`DELETE FROM enrollment_requests WHERE COALESCE(resolved_at, created_at) < ?`), before.UnixMilli())
+ if err != nil {
+ return 0, err
+ }
+ regular, err := res.RowsAffected()
+ if err != nil {
+ return 0, err
+ }
+ res, err = s.db.ExecContext(ctx, s.rebind(`DELETE FROM enrollment_requests WHERE COALESCE(resolved_at, created_at) < ? AND token_id IN (SELECT id FROM bootstrap_tokens WHERE ephemeral = ?)`), ephemeralBefore.UnixMilli(), true)
+ if err != nil {
+ return regular, err
+ }
+ ephemeral, err := res.RowsAffected()
+ return regular + ephemeral, err
+}
+
+// DeleteExpiredBootstrapTokens implements Store. The NOT EXISTS keeps the
+// foreign key from enrollment_requests.token_id satisfied: a token goes once
+// its requests have.
+func (s *SQLStore) DeleteExpiredBootstrapTokens(ctx context.Context, before time.Time) (int64, error) {
+ res, err := s.db.ExecContext(ctx, s.rebind(`DELETE FROM bootstrap_tokens WHERE expires_at < ? AND NOT EXISTS (SELECT 1 FROM enrollment_requests r WHERE r.token_id = bootstrap_tokens.id)`), before.UnixMilli())
+ if err != nil {
+ return 0, err
+ }
+ return res.RowsAffected()
+}
+
+// DeleteExpiredRouterLeases implements Store.
+func (s *SQLStore) DeleteExpiredRouterLeases(ctx context.Context, before time.Time) (int64, error) {
+ res, err := s.db.ExecContext(ctx, s.rebind(`DELETE FROM routers WHERE expires_at < ?`), before.UnixMilli())
+ if err != nil {
+ return 0, err
+ }
+ return res.RowsAffected()
+}
+
+// countedTables are the tables whose row counts the control plane reports;
+// every one of them grows with mesh activity and has a reaper or a bound.
+var countedTables = []string{"nodes", "enrollment_requests", "bootstrap_tokens", "routers", "revoked_biscuits", "banned_identities", "users", "keyring"}
+
+// CountRows implements Store.
+func (s *SQLStore) CountRows(ctx context.Context) (map[string]int64, error) {
+ counts := make(map[string]int64, len(countedTables))
+ for _, table := range countedTables {
+ var n int64
+ if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM `+table).Scan(&n); err != nil {
+ return nil, fmt.Errorf("count %s: %w", table, err)
+ }
+ counts[table] = n
+ }
+ return counts, nil
+}
+
// ListBannedPeerIDs implements Store.
func (s *SQLStore) ListBannedPeerIDs(ctx context.Context) ([]string, error) {
query := s.rebind(`SELECT peer_id FROM nodes WHERE banned = ?`)
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index 62d96592..52ad4730 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -385,6 +385,28 @@ type Store interface {
// regular and how many ephemeral rows went.
DeleteSilentBootstrapNodes(ctx context.Context, before, ephemeralBefore time.Time) (regular, ephemeral int64, err error)
+ // DeleteStaleEnrollmentRequests removes enrollment requests created, or
+ // resolved, before `before`, and before `ephemeralBefore` when the token
+ // that made them is ephemeral. A resolved request is a record of a
+ // decision; a pending one that old was abandoned by its enrollee long
+ // ago. It returns how many went.
+ DeleteStaleEnrollmentRequests(ctx context.Context, before, ephemeralBefore time.Time) (int64, error)
+
+ // DeleteExpiredBootstrapTokens removes tokens that expired before
+ // `before` and that no enrollment request refers to any more; a token
+ // is the proof a member gave once, and once its requests are gone
+ // nothing points at it. It returns how many went.
+ DeleteExpiredBootstrapTokens(ctx context.Context, before time.Time) (int64, error)
+
+ // DeleteExpiredRouterLeases removes router leases that expired before
+ // `before`. A router that comes back leases again under its peer ID; a
+ // lease long expired is a router that is gone. It returns how many went.
+ DeleteExpiredRouterLeases(ctx context.Context, before time.Time) (int64, error)
+
+ // CountRows counts the rows of every table whose size the control plane
+ // reports, keyed by table name.
+ CountRows(ctx context.Context) (map[string]int64, error)
+
// ListBootstrapTokens retrieves all bootstrap tokens.
ListBootstrapTokens(ctx context.Context) ([]BootstrapToken, error)
diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.md
index 385ddd3a..b0d0d50f 100644
--- a/site/content/docs/reference/control-plane.md
+++ b/site/content/docs/reference/control-plane.md
@@ -158,9 +158,21 @@ with `403`). The console uses these routes.
identity on every restart, so without this the node table would only grow.
`agentmesh_control_plane_nodes_deleted_total{reason}` counts the deletions:
`session`, `silent`, `ephemeral`.
+- **The records around an enrollment are garbage-collected with it.** The
+ same sweep deletes enrollment requests resolved, or left pending, for
+ `--node-retention` (for `--ephemeral-node-retention` when an ephemeral
+ token made them), bootstrap tokens that expired `--node-retention` ago and
+ that no remaining request refers to, and router leases that expired
+ `--node-retention` ago. It also drops the service report held in memory
+ for every node that is no longer admitted.
+ `agentmesh_control_plane_storage_rows_deleted_total{table}` counts these
+ deletions; `agentmesh_control_plane_storage_rows{table}` is the size of
+ each table, and a table that keeps growing past its retention is a sweep
+ that is failing, which the log says.
## Metrics
`/metrics` exposes `agentmesh_control_plane_*` counters and gauges: enrolled nodes
by role and state, active routers, mesh-connected peers as reported by
-router leases, per-route request counts and latencies, and the Go runtime.
+router leases, rows per store table, service reports held in memory,
+per-route request counts and latencies, and the Go runtime.
diff --git a/tests/integration/node_lifecycle_test.go b/tests/integration/node_lifecycle_test.go
index 7687c954..3eb2f8a0 100644
--- a/tests/integration/node_lifecycle_test.go
+++ b/tests/integration/node_lifecycle_test.go
@@ -448,6 +448,41 @@ func TestSilentEphemeralMembersAreForgotten(t *testing.T) {
t.Errorf("nodes_deleted_total{reason=silent} = %v (present %v), want 0: the regular members are within their retention", v, ok)
}
+ // The fleet's enrollment requests go with its rows: auto-approval wrote
+ // one per member, and a table of approved requests from members long
+ // gone is what the testnet accumulated a hundred thousand of. The two
+ // regular members' requests are within their retention and stay.
+ if v, ok := controlPlaneMetric(t, cpURL, `agentmesh_control_plane_storage_rows_deleted_total{table="enrollment_requests"}`); !ok || v != fleet {
+ t.Errorf("storage_rows_deleted_total{table=enrollment_requests} = %v (present %v), want %d", v, ok, fleet)
+ }
+ requests := fetchAdminEnrollments(t, cpURL, testAdminToken)
+ if len(requests) != 2 {
+ var left []string
+ for _, r := range requests {
+ left = append(left, r.GetPeerId())
+ }
+ t.Errorf("%d enrollment requests remain after the sweep (%v), want the two regular members' only", len(requests), left)
+ }
+ for _, r := range requests {
+ if r.GetPeerId() != silent.peerID.String() && r.GetPeerId() != refreshing.peerID.String() {
+ t.Errorf("enrollment request of %s survived the sweep; it belongs to the forgotten fleet", r.GetPeerId())
+ }
+ }
+ // Both tokens are within their hour; no lease was ever taken.
+ for _, series := range []string{
+ `agentmesh_control_plane_storage_rows_deleted_total{table="bootstrap_tokens"}`,
+ `agentmesh_control_plane_storage_rows_deleted_total{table="routers"}`,
+ } {
+ if v, ok := controlPlaneMetric(t, cpURL, series); !ok || v != 0 {
+ t.Errorf("%s = %v (present %v), want 0", series, v, ok)
+ }
+ }
+ for _, table := range []string{"nodes", "enrollment_requests", "bootstrap_tokens", "routers"} {
+ if _, ok := controlPlaneMetric(t, cpURL, fmt.Sprintf(`agentmesh_control_plane_storage_rows{table="%s"}`, table)); !ok {
+ t.Errorf("/metrics lacks agentmesh_control_plane_storage_rows{table=%q}", table)
+ }
+ }
+
// A forgotten member is a stranger: its credential still verifies, its
// row is gone, so it is told to enroll again. The regular ones refresh.
if status, body := members[0].refresh(t, cpURL); status != http.StatusUnauthorized {
@@ -460,6 +495,31 @@ func TestSilentEphemeralMembersAreForgotten(t *testing.T) {
}
}
+// fetchAdminEnrollments lists every enrollment request the control plane
+// still holds.
+func fetchAdminEnrollments(t *testing.T, cpURL, adminToken string) []*api.EnrollmentRequest {
+ t.Helper()
+ req, err := http.NewRequest(http.MethodGet, cpURL+"/admin/enrollments", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Authorization", "Bearer "+adminToken)
+ resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req)
+ if err != nil {
+ t.Fatalf("GET /admin/enrollments: %v", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ data, _ := io.ReadAll(resp.Body)
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("GET /admin/enrollments: %d %s", resp.StatusCode, data)
+ }
+ var list api.EnrollmentRequestListResponse
+ if err := protojson.Unmarshal(data, &list); err != nil {
+ t.Fatalf("decode /admin/enrollments: %v", err)
+ }
+ return list.GetRequests()
+}
+
// mintBootstrapTokenWithUses mints a node bootstrap token good for uses
// enrollments, ephemeral or not.
func mintBootstrapTokenWithUses(t *testing.T, cpURL, adminToken string, uses int, ephemeral bool) string {