Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane: the sweep bounds every table enrollment grows, and the catalog cache
Antonio Ojeab9ae58fab079
diff --git a/internal/controlplane/catalog.go b/internal/controlplane/catalog.goindex 820b703e..ab79b0ac 100644--- a/internal/controlplane/catalog.go+++ b/internal/controlplane/catalog.go@@ -93,6 +93,7 @@ func (s *Server) dropCatalogEntry(peerID string) {}s.catalogMu.Lock()delete(s.catalog, pID.String())+ catalogEntries.Set(float64(len(s.catalog)))s.catalogMu.Unlock()}@@ -144,6 +145,7 @@ func (s *Server) HandleNodeCatalog(w http.ResponseWriter, r *http.Request) {Services: req.Services,ReportedAt: time.Now(),}+ catalogEntries.Set(float64(len(s.catalog)))s.catalogMu.Unlock()w.WriteHeader(http.StatusNoContent)diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.goindex fb1f2a84..29adf675 100644--- a/internal/controlplane/metrics.go+++ b/internal/controlplane/metrics.go@@ -113,6 +113,10 @@ type meshSnapshot struct {// the retiring one stops verifying; zero when there is none.signingKeys intretiringKeyExpiry time.Time+ // storageRows is the row count of every table the control plane keeps+ // bounded; a table that grows past its retention is a reaper that+ // stopped working.+ storageRows map[string]int64}// meshStateCollector exports the control plane's view of the mesh: who is@@ -141,6 +145,7 @@ type meshStateCollector struct {meshPeersDesc *prometheus.DescsigningKeysDesc *prometheus.DescretiringKeyDesc *prometheus.Desc+ storageRowsDesc *prometheus.DescscrapeOKDesc *prometheus.DescscrapeSampleDesc *prometheus.Desc}@@ -195,6 +200,10 @@ func newMeshStateCollector(store storage.Store) *meshStateCollector {"agentmesh_control_plane_retiring_key_expiry_timestamp_seconds","Unix time the retiring signing key stops verifying; absent when no rotation is in its grace period",nil, nil),+ storageRowsDesc: prometheus.NewDesc(+ "agentmesh_control_plane_storage_rows",+ "Rows in a control plane store table",+ []string{"table"}, nil),scrapeOKDesc: prometheus.NewDesc("agentmesh_control_plane_mesh_state_scrape_success","1 if the mesh state was read from the store, 0 if the last read failed",@@ -210,7 +219,7 @@ func (c *meshStateCollector) Describe(ch chan<- *prometheus.Desc) {for _, d := range []*prometheus.Desc{c.nodesDesc, c.usersDesc, c.requestsDesc, c.tokensDesc, c.routersDesc,c.routerPeersDesc, c.routerDHTDesc, c.routerLeaseDesc, c.meshPeersDesc,- c.signingKeysDesc, c.retiringKeyDesc, c.scrapeOKDesc, c.scrapeSampleDesc,+ c.signingKeysDesc, c.retiringKeyDesc, c.storageRowsDesc, c.scrapeOKDesc, c.scrapeSampleDesc,} {ch <- d}@@ -257,6 +266,9 @@ func (c *meshStateCollector) Collect(ch chan<- prometheus.Metric) {if !s.retiringKeyExpiry.IsZero() {ch <- prometheus.MustNewConstMetric(c.retiringKeyDesc, prometheus.GaugeValue, float64(s.retiringKeyExpiry.Unix()))}+ for table, n := range s.storageRows {+ ch <- prometheus.MustNewConstMetric(c.storageRowsDesc, prometheus.GaugeValue, float64(n), table)+ }}// refresh replaces the snapshot from the store. A failed read keeps the last@@ -335,6 +347,11 @@ func readMeshSnapshot(ctx context.Context, store storage.Store, now time.Time) (}}+ snap.storageRows, err = store.CountRows(ctx)+ if err != nil {+ return snap, err+ }+return snap, nil}diff --git a/internal/controlplane/metrics_test.go b/internal/controlplane/metrics_test.goindex a7b0044c..099d97d7 100644--- a/internal/controlplane/metrics_test.go+++ b/internal/controlplane/metrics_test.go@@ -343,6 +343,10 @@ func TestMetricsEndpointServesMeshStateAndRuntime(t *testing.T) {"agentmesh_control_plane_mesh_state_scrape_success 1","agentmesh_control_plane_routers_active 0","agentmesh_control_plane_mesh_connected_peers 0",+ `agentmesh_control_plane_storage_rows{table="nodes"} 0`,+ `agentmesh_control_plane_storage_rows{table="enrollment_requests"} 0`,+ "agentmesh_control_plane_catalog_entries ",+ `agentmesh_control_plane_storage_rows_deleted_total{table="routers"} `,"go_goroutines",} {if !strings.Contains(string(body), want) {diff --git a/internal/controlplane/node_gc.go b/internal/controlplane/node_gc.goindex 83a32d9a..8056f15f 100644--- a/internal/controlplane/node_gc.go+++ b/internal/controlplane/node_gc.go@@ -20,6 +20,8 @@ import ("github.com/prometheus/client_golang/prometheus""github.com/prometheus/client_golang/prometheus/promauto"++ "github.com/google/agentmesh/internal/storage")const (@@ -60,31 +62,136 @@ const (deletedReasonEphemeral = "ephemeral")+var storageRowsDeletedTotal = promauto.NewCounterVec(prometheus.CounterOpts{+ Name: "agentmesh_control_plane_storage_rows_deleted_total",+ Help: "Rows the storage GC removed from a table other than nodes: enrollment requests older than the retention, bootstrap tokens expired for the retention that no request refers to, router leases expired for the retention",+}, []string{"table"})++var catalogEntries = promauto.NewGauge(prometheus.GaugeOpts{+ Name: "agentmesh_control_plane_catalog_entries",+ Help: "Service reports the control plane holds in memory, one per node that reported since it was admitted",+})++const (+ tableEnrollmentRequests = "enrollment_requests"+ tableBootstrapTokens = "bootstrap_tokens"+ tableRouters = "routers"+)+func init() {for _, reason := range []string{deletedReasonSession, deletedReasonSilent, deletedReasonEphemeral} {nodesDeletedTotal.WithLabelValues(reason)}+ for _, table := range []string{tableEnrollmentRequests, tableBootstrapTokens, tableRouters} {+ storageRowsDeletedTotal.WithLabelValues(table)+ }}// runNodeGCLoop sweeps once at start and then hourly. Replicas each sweep-// against the shared database; the delete is idempotent, so the race is+// against the shared database; the deletes are idempotent, so the race is// harmless and not worth a claim.func (s *Server) runNodeGCLoop() {defer s.wg.Done()- s.gcExpiredNodes(time.Now())+ s.gcStorage(time.Now())ticker := time.NewTicker(nodeGCInterval(s.config.NodeRetention))defer ticker.Stop()for {select {case <-ticker.C:- s.gcExpiredNodes(time.Now())+ s.gcStorage(time.Now())case <-s.ctx.Done():return}}}+// gcStorage is one sweep over everything that grows with mesh activity: the+// node records, the enrollment requests, the bootstrap tokens, the router+// leases, and the in-memory service catalog. Each step logs its own failure+// and the next one still runs, so one table's trouble does not let another+// grow.+func (s *Server) gcStorage(now time.Time) {+ s.gcExpiredNodes(now)+ s.gcEnrollmentRecords(now)+ s.pruneCatalog(now)+}++// gcEnrollmentRecords deletes the enrollment requests resolved, or left+// pending, more than NodeRetention ago (EphemeralNodeRetention when an+// ephemeral token made them), then the bootstrap tokens that expired more+// than NodeRetention ago and that no remaining request refers to, then the+// router leases expired for as long. Requests go before tokens because a+// request refers to its token.+func (s *Server) gcEnrollmentRecords(now time.Time) {+ ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)+ defer cancel()++ before := now.Add(-s.config.NodeRetention)+ requests, err := s.store.DeleteStaleEnrollmentRequests(ctx, before, now.Add(-s.config.EphemeralNodeRetention))+ if err != nil {+ logger.Errorf("Failed to delete stale enrollment requests: %v", err)+ } else if requests > 0 {+ storageRowsDeletedTotal.WithLabelValues(tableEnrollmentRequests).Add(float64(requests))+ logger.Infof("Deleted %d enrollment requests older than the retention", requests)+ }++ tokens, err := s.store.DeleteExpiredBootstrapTokens(ctx, before)+ if err != nil {+ logger.Errorf("Failed to delete expired bootstrap tokens: %v", err)+ } else if tokens > 0 {+ storageRowsDeletedTotal.WithLabelValues(tableBootstrapTokens).Add(float64(tokens))+ logger.Infof("Deleted %d bootstrap tokens expired more than %s ago", tokens, s.config.NodeRetention)+ }++ leases, err := s.store.DeleteExpiredRouterLeases(ctx, before)+ if err != nil {+ logger.Errorf("Failed to delete expired router leases: %v", err)+ } else if leases > 0 {+ storageRowsDeletedTotal.WithLabelValues(tableRouters).Add(float64(leases))+ logger.Infof("Deleted %d router leases expired more than %s ago", leases, s.config.NodeRetention)+ }+}++// pruneCatalog forgets the service report of every peer that is no longer+// an admitted node: its enrollment lapsed, was collected or was banned on+// another replica. The console already hides such reports; this keeps the+// cache from holding one entry per member that ever joined until the next+// restart.+func (s *Server) pruneCatalog(now time.Time) {+ ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)+ defer cancel()++ snap := s.catalogSnapshot()+ if len(snap) == 0 {+ catalogEntries.Set(0)+ return+ }+ admitted, err := s.store.ListNodes(ctx, storage.NodeFilter{States: []storage.NodeState{storage.NodeAdmitted}, Now: now})+ if err != nil {+ logger.Errorf("Failed to list admitted nodes to prune the service catalog: %v", err)+ return+ }+ keep := make(map[string]struct{}, len(admitted))+ for i := range admitted {+ keep[admitted[i].PeerID] = struct{}{}+ }+ dropped := 0+ s.catalogMu.Lock()+ for peerID := range snap {+ if _, ok := keep[peerID]; !ok {+ delete(s.catalog, peerID)+ dropped+++ }+ }+ remaining := len(s.catalog)+ s.catalogMu.Unlock()+ catalogEntries.Set(float64(remaining))+ if dropped > 0 {+ logger.Infof("Dropped %d service reports of nodes no longer admitted, %d remain", dropped, remaining)+ }+}+// gcExpiredNodes deletes every unbanned node whose session expired more than// NodeRetention before now, and every unbanned bootstrap member, which has// no session, that did not refresh for NodeRetention, or fordiff --git a/internal/controlplane/node_gc_test.go b/internal/controlplane/node_gc_test.goindex f504ba6f..b8573556 100644--- a/internal/controlplane/node_gc_test.go+++ b/internal/controlplane/node_gc_test.go@@ -21,6 +21,8 @@ import ("testing""time"+ dto "github.com/prometheus/client_model/go"+"github.com/google/agentmesh/api""github.com/google/agentmesh/internal/storage")@@ -139,6 +141,102 @@ func TestGCForgetsSilentBootstrapMembers(t *testing.T) {}}+// One sweep also bounds the records enrollment leaves behind: the requests,+// the tokens that made them, the router leases, and the in-memory service+// catalog. Every table the testnet audit found growing is covered here.+func TestGCStorageBoundsEnrollmentRecordsAndCatalog(t *testing.T) {+ store := newMetricsTestStore(t)+ ctx := context.Background()+ now := time.Now()+ day := 24 * time.Hour++ if err := store.SaveBootstrapToken(ctx, &storage.BootstrapToken{+ ID: "old", TokenHash: "hash-old", Role: api.RoleNode, MaxUsages: 10, CreatedAt: now.Add(-41 * day), ExpiresAt: now.Add(-40 * day),+ }); err != nil {+ t.Fatal(err)+ }+ if err := store.SaveBootstrapToken(ctx, &storage.BootstrapToken{+ ID: "live", TokenHash: "hash-live", Role: api.RoleNode, MaxUsages: 10, CreatedAt: now, ExpiresAt: now.Add(day),+ }); err != nil {+ t.Fatal(err)+ }+ resolved := now.Add(-40 * day)+ if err := store.CreateEnrollmentRequest(ctx, &storage.EnrollmentRequest{+ ID: "old-request", PeerID: "old-peer", PublicKey: []byte("pub"), TokenID: "old",+ Status: api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED, CreatedAt: now.Add(-41 * day), ResolvedAt: &resolved,+ }); err != nil {+ t.Fatal(err)+ }+ if err := store.CreateEnrollmentRequest(ctx, &storage.EnrollmentRequest{+ ID: "live-request", PeerID: "live-peer", PublicKey: []byte("pub"), TokenID: "live",+ Status: api.EnrollmentStatus_ENROLLMENT_STATUS_PENDING, CreatedAt: now,+ }); err != nil {+ t.Fatal(err)+ }+ for _, lease := range []struct {+ id string+ expiresAt time.Time+ }{{"router-live", now.Add(time.Minute)}, {"router-gone", now.Add(-40 * day)}} {+ if err := store.UpsertRouterLease(ctx, &storage.RouterLease{PeerID: lease.id, Addresses: []string{"/ip4/127.0.0.1/tcp/1"}, LastRenewal: now, ExpiresAt: lease.expiresAt}); err != nil {+ t.Fatal(err)+ }+ }+ // An admitted node that reported services, and a report left by a node+ // whose row the sweep already collected.+ admitted, gone := "admitted-peer", "gone-peer"+ if err := store.EnrollNode(ctx, &storage.EnrolledNode{+ PeerID: admitted, PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode, EnrollmentType: "BOOTSTRAP", EnrolledAt: now,+ }); err != nil {+ t.Fatal(err)+ }++ srv, err := NewServer(Options{+ DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: "https://issuer",+ NodeRetention: 30 * day, EphemeralNodeRetention: time.Hour,+ }, store)+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = srv.Close() }()+ srv.catalog[admitted] = nodeCatalogEntry{ReportedAt: now}+ srv.catalog[gone] = nodeCatalogEntry{ReportedAt: now}++ before := map[string]float64{}+ for _, table := range []string{tableEnrollmentRequests, tableBootstrapTokens, tableRouters} {+ before[table] = counterValue(t, storageRowsDeletedTotal, table)+ }++ srv.gcStorage(now)++ for table, want := range map[string]float64{tableEnrollmentRequests: 1, tableBootstrapTokens: 1, tableRouters: 1} {+ if got := counterValue(t, storageRowsDeletedTotal, table) - before[table]; got != want {+ t.Errorf("storage_rows_deleted_total{table=%s} advanced by %v, want %v", table, got, want)+ }+ }+ counts, err := store.CountRows(ctx)+ if err != nil {+ t.Fatal(err)+ }+ for table, want := range map[string]int64{"enrollment_requests": 1, "bootstrap_tokens": 1, "routers": 1, "nodes": 1} {+ if counts[table] != want {+ t.Errorf("%s has %d rows after the sweep, want %d", table, counts[table], want)+ }+ }+ if _, ok := srv.catalogSnapshot()[gone]; ok {+ t.Errorf("catalog still holds the report of a node that is no longer enrolled")+ }+ if _, ok := srv.catalogSnapshot()[admitted]; !ok {+ t.Errorf("catalog dropped the report of an admitted node")+ }+ var m dto.Metric+ if err := catalogEntries.Write(&m); err != nil {+ t.Fatal(err)+ }+ if got := m.GetGauge().GetValue(); got != 1 {+ t.Errorf("catalog_entries = %v, want 1", got)+ }+}+// A retention that is not positive is refused, not read as "keep forever":// an unbounded nodes table is what this GC exists to prevent. Unset is the// default (TestGCExpiredNodesDefaultsRetention).diff --git a/internal/storage/round_trip_test.go b/internal/storage/round_trip_test.goindex 2617f021..721fd46f 100644--- a/internal/storage/round_trip_test.go+++ b/internal/storage/round_trip_test.go@@ -303,6 +303,143 @@ func TestDeleteSilentBootstrapNodes(t *testing.T) {}}+func TestDeleteStaleEnrollmentRecords(t *testing.T) {+ store := newTestStore(t)+ ctx := context.Background()+ now := time.Now()+ day := 24 * time.Hour+ token := func(id string, expiresAt time.Time, ephemeral bool) {+ t.Helper()+ if err := store.SaveBootstrapToken(ctx, &BootstrapToken{+ ID: id, TokenHash: "hash-" + id, Role: api.RoleNode, MaxUsages: 10,+ CreatedAt: expiresAt.Add(-day), ExpiresAt: expiresAt, Ephemeral: ephemeral,+ }); err != nil {+ t.Fatalf("token %s: %v", id, err)+ }+ }+ request := func(id, tokenID string, createdAt time.Time, resolvedAt *time.Time) {+ t.Helper()+ if err := store.CreateEnrollmentRequest(ctx, &EnrollmentRequest{+ ID: id, PeerID: "peer-" + id, PublicKey: []byte("pub"), TokenID: tokenID,+ Status: api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED, CreatedAt: createdAt, ResolvedAt: resolvedAt,+ }); err != nil {+ t.Fatalf("request %s: %v", id, err)+ }+ }+ at := func(ago time.Duration) *time.Time { ts := now.Add(-ago); return &ts }++ // A token still in use, one that expired long ago but whose request is+ // recent, one expired long ago with an old request, an ephemeral one+ // whose requests are two hours old, and one that just expired.+ token("live", now.Add(day), false)+ token("old-with-recent-request", now.Add(-40*day), false)+ token("old", now.Add(-40*day), false)+ token("ephemeral", now.Add(-2*time.Hour), true)+ token("just-expired", now.Add(-time.Hour), false)+ request("live-1", "live", now.Add(-time.Hour), at(time.Hour))+ request("recent-on-old-token", "old-with-recent-request", now.Add(-41*day), at(2*day))+ request("old-resolved", "old", now.Add(-41*day), at(40*day))+ request("old-pending", "old", now.Add(-41*day), nil)+ request("ephemeral-1", "ephemeral", now.Add(-2*time.Hour), at(2*time.Hour))+ request("ephemeral-2", "ephemeral", now.Add(-2*time.Hour), at(2*time.Hour))+ request("ephemeral-fresh", "ephemeral", now.Add(-10*time.Minute), at(10*time.Minute))++ requests, err := store.DeleteStaleEnrollmentRequests(ctx, now.Add(-30*day), now.Add(-time.Hour))+ if err != nil {+ t.Fatalf("DeleteStaleEnrollmentRequests: %v", err)+ }+ if requests != 4 {+ t.Errorf("deleted %d requests, want 4 (old-resolved, old-pending, ephemeral-1, ephemeral-2)", requests)+ }+ left, err := store.ListEnrollmentRequests(ctx)+ if err != nil {+ t.Fatal(err)+ }+ var leftIDs []string+ for _, r := range left {+ leftIDs = append(leftIDs, r.ID)+ }+ slices.Sort(leftIDs)+ if want := []string{"ephemeral-fresh", "live-1", "recent-on-old-token"}; !slices.Equal(leftIDs, want) {+ t.Errorf("remaining requests = %v, want %v", leftIDs, want)+ }++ // "old" lost both requests and goes; "old-with-recent-request" is still+ // referred to and stays; "ephemeral" keeps its fresh request; "just-expired"+ // is within the retention.+ tokens, err := store.DeleteExpiredBootstrapTokens(ctx, now.Add(-30*day))+ if err != nil {+ t.Fatalf("DeleteExpiredBootstrapTokens: %v", err)+ }+ if tokens != 1 {+ t.Errorf("deleted %d tokens, want 1 (old)", tokens)+ }+ remaining, err := store.ListBootstrapTokens(ctx)+ if err != nil {+ t.Fatal(err)+ }+ var tokenIDs []string+ for i := range remaining {+ tokenIDs = append(tokenIDs, remaining[i].ID)+ }+ slices.Sort(tokenIDs)+ if want := []string{"ephemeral", "just-expired", "live", "old-with-recent-request"}; !slices.Equal(tokenIDs, want) {+ t.Errorf("remaining tokens = %v, want %v", tokenIDs, want)+ }+}++func TestDeleteExpiredRouterLeases(t *testing.T) {+ store := newTestStore(t)+ ctx := context.Background()+ now := time.Now()+ lease := func(id string, expiresAt time.Time) {+ t.Helper()+ if err := store.UpsertRouterLease(ctx, &RouterLease{+ PeerID: id, Addresses: []string{"/ip4/127.0.0.1/tcp/1"}, LastRenewal: expiresAt.Add(-time.Minute), ExpiresAt: expiresAt,+ }); err != nil {+ t.Fatalf("lease %s: %v", id, err)+ }+ }+ lease("live", now.Add(time.Minute))+ lease("lapsed-today", now.Add(-time.Hour))+ lease("gone", now.Add(-40*24*time.Hour))++ deleted, err := store.DeleteExpiredRouterLeases(ctx, now.Add(-30*24*time.Hour))+ if err != nil {+ t.Fatalf("DeleteExpiredRouterLeases: %v", err)+ }+ if deleted != 1 {+ t.Errorf("deleted %d leases, want 1 (gone)", deleted)+ }+ counts, err := store.CountRows(ctx)+ if err != nil {+ t.Fatalf("CountRows: %v", err)+ }+ if counts["routers"] != 2 {+ t.Errorf("routers rows = %d, want 2 (live, lapsed-today); counts %v", counts["routers"], counts)+ }+ active, err := store.GetActiveRouters(ctx)+ if err != nil {+ t.Fatal(err)+ }+ if len(active) != 1 || active[0].PeerID != "live" {+ t.Errorf("active routers = %+v, want only live", active)+ }+}++func TestCountRowsCoversEveryGrowingTable(t *testing.T) {+ store := newTestStore(t)+ counts, err := store.CountRows(context.Background())+ if err != nil {+ t.Fatalf("CountRows: %v", err)+ }+ for _, table := range []string{"nodes", "enrollment_requests", "bootstrap_tokens", "routers", "revoked_biscuits", "banned_identities", "users", "keyring"} {+ if _, ok := counts[table]; !ok {+ t.Errorf("CountRows lacks %q: %v", table, counts)+ }+ }+}+func TestDeleteExpiredNodesSparesBannedAndUnbounded(t *testing.T) {store := newTestStore(t)ctx := context.Background()diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.goindex 9d4ea76e..498fc4e9 100644--- a/internal/storage/sql_store.go+++ b/internal/storage/sql_store.go@@ -1122,6 +1122,61 @@ func (s *SQLStore) DeleteSilentBootstrapNodes(ctx context.Context, before, ephemreturn regular, ephemeral, err}+// DeleteStaleEnrollmentRequests implements Store.+func (s *SQLStore) DeleteStaleEnrollmentRequests(ctx context.Context, before, ephemeralBefore time.Time) (int64, error) {+ res, err := s.db.ExecContext(ctx, s.rebind(`DELETE FROM enrollment_requests WHERE COALESCE(resolved_at, created_at) < ?`), before.UnixMilli())+ if err != nil {+ return 0, err+ }+ regular, err := res.RowsAffected()+ if err != nil {+ return 0, err+ }+ res, err = s.db.ExecContext(ctx, s.rebind(`DELETE FROM enrollment_requests WHERE COALESCE(resolved_at, created_at) < ? AND token_id IN (SELECT id FROM bootstrap_tokens WHERE ephemeral = ?)`), ephemeralBefore.UnixMilli(), true)+ if err != nil {+ return regular, err+ }+ ephemeral, err := res.RowsAffected()+ return regular + ephemeral, err+}++// DeleteExpiredBootstrapTokens implements Store. The NOT EXISTS keeps the+// foreign key from enrollment_requests.token_id satisfied: a token goes once+// its requests have.+func (s *SQLStore) DeleteExpiredBootstrapTokens(ctx context.Context, before time.Time) (int64, error) {+ res, err := s.db.ExecContext(ctx, s.rebind(`DELETE FROM bootstrap_tokens WHERE expires_at < ? AND NOT EXISTS (SELECT 1 FROM enrollment_requests r WHERE r.token_id = bootstrap_tokens.id)`), before.UnixMilli())+ if err != nil {+ return 0, err+ }+ return res.RowsAffected()+}++// DeleteExpiredRouterLeases implements Store.+func (s *SQLStore) DeleteExpiredRouterLeases(ctx context.Context, before time.Time) (int64, error) {+ res, err := s.db.ExecContext(ctx, s.rebind(`DELETE FROM routers WHERE expires_at < ?`), before.UnixMilli())+ if err != nil {+ return 0, err+ }+ return res.RowsAffected()+}++// countedTables are the tables whose row counts the control plane reports;+// every one of them grows with mesh activity and has a reaper or a bound.+var countedTables = []string{"nodes", "enrollment_requests", "bootstrap_tokens", "routers", "revoked_biscuits", "banned_identities", "users", "keyring"}++// CountRows implements Store.+func (s *SQLStore) CountRows(ctx context.Context) (map[string]int64, error) {+ counts := make(map[string]int64, len(countedTables))+ for _, table := range countedTables {+ var n int64+ if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM `+table).Scan(&n); err != nil {+ return nil, fmt.Errorf("count %s: %w", table, err)+ }+ counts[table] = n+ }+ return counts, nil+}+// ListBannedPeerIDs implements Store.func (s *SQLStore) ListBannedPeerIDs(ctx context.Context) ([]string, error) {query := s.rebind(`SELECT peer_id FROM nodes WHERE banned = ?`)diff --git a/internal/storage/storage.go b/internal/storage/storage.goindex 62d96592..52ad4730 100644--- a/internal/storage/storage.go+++ b/internal/storage/storage.go@@ -385,6 +385,28 @@ type Store interface {// regular and how many ephemeral rows went.DeleteSilentBootstrapNodes(ctx context.Context, before, ephemeralBefore time.Time) (regular, ephemeral int64, err error)+ // DeleteStaleEnrollmentRequests removes enrollment requests created, or+ // resolved, before `before`, and before `ephemeralBefore` when the token+ // that made them is ephemeral. A resolved request is a record of a+ // decision; a pending one that old was abandoned by its enrollee long+ // ago. It returns how many went.+ DeleteStaleEnrollmentRequests(ctx context.Context, before, ephemeralBefore time.Time) (int64, error)++ // DeleteExpiredBootstrapTokens removes tokens that expired before+ // `before` and that no enrollment request refers to any more; a token+ // is the proof a member gave once, and once its requests are gone+ // nothing points at it. It returns how many went.+ DeleteExpiredBootstrapTokens(ctx context.Context, before time.Time) (int64, error)++ // DeleteExpiredRouterLeases removes router leases that expired before+ // `before`. A router that comes back leases again under its peer ID; a+ // lease long expired is a router that is gone. It returns how many went.+ DeleteExpiredRouterLeases(ctx context.Context, before time.Time) (int64, error)++ // CountRows counts the rows of every table whose size the control plane+ // reports, keyed by table name.+ CountRows(ctx context.Context) (map[string]int64, error)+// ListBootstrapTokens retrieves all bootstrap tokens.ListBootstrapTokens(ctx context.Context) ([]BootstrapToken, error)diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.mdindex 385ddd3a..b0d0d50f 100644--- a/site/content/docs/reference/control-plane.md+++ b/site/content/docs/reference/control-plane.md@@ -158,9 +158,21 @@ with `403`). The console uses these routes.identity on every restart, so without this the node table would only grow.`agentmesh_control_plane_nodes_deleted_total{reason}` counts the deletions:`session`, `silent`, `ephemeral`.+- **The records around an enrollment are garbage-collected with it.** The+ same sweep deletes enrollment requests resolved, or left pending, for+ `--node-retention` (for `--ephemeral-node-retention` when an ephemeral+ token made them), bootstrap tokens that expired `--node-retention` ago and+ that no remaining request refers to, and router leases that expired+ `--node-retention` ago. It also drops the service report held in memory+ for every node that is no longer admitted.+ `agentmesh_control_plane_storage_rows_deleted_total{table}` counts these+ deletions; `agentmesh_control_plane_storage_rows{table}` is the size of+ each table, and a table that keeps growing past its retention is a sweep+ that is failing, which the log says.## Metrics`/metrics` exposes `agentmesh_control_plane_*` counters and gauges: enrolled nodesby role and state, active routers, mesh-connected peers as reported by-router leases, per-route request counts and latencies, and the Go runtime.+router leases, rows per store table, service reports held in memory,+per-route request counts and latencies, and the Go runtime.diff --git a/tests/integration/node_lifecycle_test.go b/tests/integration/node_lifecycle_test.goindex 7687c954..3eb2f8a0 100644--- a/tests/integration/node_lifecycle_test.go+++ b/tests/integration/node_lifecycle_test.go@@ -448,6 +448,41 @@ func TestSilentEphemeralMembersAreForgotten(t *testing.T) {t.Errorf("nodes_deleted_total{reason=silent} = %v (present %v), want 0: the regular members are within their retention", v, ok)}+ // The fleet's enrollment requests go with its rows: auto-approval wrote+ // one per member, and a table of approved requests from members long+ // gone is what the testnet accumulated a hundred thousand of. The two+ // regular members' requests are within their retention and stay.+ if v, ok := controlPlaneMetric(t, cpURL, `agentmesh_control_plane_storage_rows_deleted_total{table="enrollment_requests"}`); !ok || v != fleet {+ t.Errorf("storage_rows_deleted_total{table=enrollment_requests} = %v (present %v), want %d", v, ok, fleet)+ }+ requests := fetchAdminEnrollments(t, cpURL, testAdminToken)+ if len(requests) != 2 {+ var left []string+ for _, r := range requests {+ left = append(left, r.GetPeerId())+ }+ t.Errorf("%d enrollment requests remain after the sweep (%v), want the two regular members' only", len(requests), left)+ }+ for _, r := range requests {+ if r.GetPeerId() != silent.peerID.String() && r.GetPeerId() != refreshing.peerID.String() {+ t.Errorf("enrollment request of %s survived the sweep; it belongs to the forgotten fleet", r.GetPeerId())+ }+ }+ // Both tokens are within their hour; no lease was ever taken.+ for _, series := range []string{+ `agentmesh_control_plane_storage_rows_deleted_total{table="bootstrap_tokens"}`,+ `agentmesh_control_plane_storage_rows_deleted_total{table="routers"}`,+ } {+ if v, ok := controlPlaneMetric(t, cpURL, series); !ok || v != 0 {+ t.Errorf("%s = %v (present %v), want 0", series, v, ok)+ }+ }+ for _, table := range []string{"nodes", "enrollment_requests", "bootstrap_tokens", "routers"} {+ if _, ok := controlPlaneMetric(t, cpURL, fmt.Sprintf(`agentmesh_control_plane_storage_rows{table="%s"}`, table)); !ok {+ t.Errorf("/metrics lacks agentmesh_control_plane_storage_rows{table=%q}", table)+ }+ }+// A forgotten member is a stranger: its credential still verifies, its// row is gone, so it is told to enroll again. The regular ones refresh.if status, body := members[0].refresh(t, cpURL); status != http.StatusUnauthorized {@@ -460,6 +495,31 @@ func TestSilentEphemeralMembersAreForgotten(t *testing.T) {}}+// fetchAdminEnrollments lists every enrollment request the control plane+// still holds.+func fetchAdminEnrollments(t *testing.T, cpURL, adminToken string) []*api.EnrollmentRequest {+ t.Helper()+ req, err := http.NewRequest(http.MethodGet, cpURL+"/admin/enrollments", nil)+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Authorization", "Bearer "+adminToken)+ resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req)+ if err != nil {+ t.Fatalf("GET /admin/enrollments: %v", err)+ }+ defer func() { _ = resp.Body.Close() }()+ data, _ := io.ReadAll(resp.Body)+ if resp.StatusCode != http.StatusOK {+ t.Fatalf("GET /admin/enrollments: %d %s", resp.StatusCode, data)+ }+ var list api.EnrollmentRequestListResponse+ if err := protojson.Unmarshal(data, &list); err != nil {+ t.Fatalf("decode /admin/enrollments: %v", err)+ }+ return list.GetRequests()+}+// mintBootstrapTokenWithUses mints a node bootstrap token good for uses// enrollments, ephemeral or not.func mintBootstrapTokenWithUses(t *testing.T, cpURL, adminToken string, uses int, ephemeral bool) string {