Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane: delete node records that outlived their session
Antonio Ojeaa4faefbd435c
diff --git a/cmd/sam-control-plane/main.go b/cmd/sam-control-plane/main.goindex 62b281d9..f9c7ae7b 100644--- a/cmd/sam-control-plane/main.go+++ b/cmd/sam-control-plane/main.go@@ -45,6 +45,7 @@ var (leaseDuration time.DurationbiscuitTTL time.DurationoidcSessionTTL time.Duration+ nodeRetention time.DurationadminTokenPath stringinsecureSkipTLSVerify boollogLevel string@@ -126,6 +127,7 @@ func main() {BiscuitTimeout: 10 * time.Second,BiscuitTTL: biscuitTTL,OIDCSessionTTL: oidcSessionTTL,+ NodeRetention: nodeRetention,AdminToken: adminToken,AutoApproveEnrollment: autoApproveEnrollment,}@@ -161,6 +163,7 @@ func main() {rootCmd.Flags().DurationVar(&leaseDuration, "lease-duration", 15*time.Minute, "Router lease registration TTL.")rootCmd.Flags().DurationVar(&biscuitTTL, "biscuit-ttl", api.BiscuitTokenTTL, "Lifespan minted into every issued Biscuit's expiration fact. Capped to the OIDC token's own expiry when shorter.")rootCmd.Flags().DurationVar(&oidcSessionTTL, "oidc-session-ttl", api.OIDCSessionTTL, "How long an OIDC enrollment stays refreshable before the identity must re-authenticate with the OIDC provider. Shorter values keep the provider authoritative for offboarding at the cost of more frequent interactive re-enrollment.")+ rootCmd.Flags().DurationVar(&nodeRetention, "node-retention", controlplane.DefaultNodeRetention, "How long an enrolled node's record is kept after its session expired before it is deleted. Banned nodes are always kept. 0 keeps every record forever.")rootCmd.Flags().StringVar(&adminTokenPath, "admin-token-path", "", "Path to file containing the token for authenticating policy REST API requests (or env SAM_ADMIN_TOKEN)")rootCmd.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", false, "Skip TLS verification for OIDC providers")rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")diff --git a/internal/controlplane/config.go b/internal/controlplane/config.goindex b090a47a..20ad86bf 100644--- a/internal/controlplane/config.go+++ b/internal/controlplane/config.go@@ -36,8 +36,13 @@ type Options struct {BiscuitTimeout time.DurationBiscuitTTL time.Duration // Lifespan minted into every issued Biscuit's expiration() fact; defaults to api.BiscuitTokenTTLOIDCSessionTTL time.Duration // How long an OIDC enrollment stays refreshable before the identity must re-authenticate interactively; defaults to api.OIDCSessionTTL- AdminToken string // Optional: administrative bearer token for protecting policy and enrollment queue REST APIs- AutoApproveEnrollment bool // If true, valid bootstrap token enrollment requests are immediately approved without administrative manual gate+ // NodeRetention is how long an enrolled node's row is kept after its+ // session expired before being deleted; 0 keeps rows forever. Every+ // pod restart without a persistent data dir enrolls a fresh identity,+ // so without this the nodes table only ever grows.+ NodeRetention time.Duration+ AdminToken string // Optional: administrative bearer token for protecting policy and enrollment queue REST APIs+ AutoApproveEnrollment bool // If true, valid bootstrap token enrollment requests are immediately approved without administrative manual gate}// Default sets default values for control plane options.diff --git a/internal/controlplane/node_gc.go b/internal/controlplane/node_gc.gonew file mode 100644index 00000000..07ff5844--- /dev/null+++ b/internal/controlplane/node_gc.go@@ -0,0 +1,83 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package controlplane++import (+ "context"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+)++const (+ // DefaultNodeRetention keeps a lapsed enrollment for a month past its+ // session, long enough to be looked up when investigating an incident,+ // and is what --node-retention defaults to.+ DefaultNodeRetention = 30 * 24 * time.Hour++ // nodeGCInterval is how often the sweep runs. Rows lapse on a scale of+ // days, so nothing is gained by sweeping more often than hourly.+ nodeGCInterval = time.Hour+)++var nodesDeletedTotal = promauto.NewCounter(prometheus.CounterOpts{+ Name: "sam_control_plane_nodes_deleted_total",+ Help: "Enrolled node records removed after outliving their session by the retention period",+})++// runNodeGCLoop sweeps once at start and then hourly. Replicas each sweep+// against the shared database; the delete is idempotent, so the race is+// harmless and not worth a claim.+func (s *Server) runNodeGCLoop() {+ defer s.wg.Done()+ if s.config.NodeRetention <= 0 {+ return+ }++ s.gcExpiredNodes(time.Now())++ ticker := time.NewTicker(nodeGCInterval)+ defer ticker.Stop()+ for {+ select {+ case <-ticker.C:+ s.gcExpiredNodes(time.Now())+ case <-s.ctx.Done():+ return+ }+ }+}++// gcExpiredNodes deletes every unbanned node whose session expired more than+// NodeRetention before now. A retention of zero means keep forever, and that+// holds here too, not only in the loop that decides whether to tick.+func (s *Server) gcExpiredNodes(now time.Time) {+ if s.config.NodeRetention <= 0 {+ return+ }+ ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)+ defer cancel()++ deleted, err := s.store.DeleteExpiredNodes(ctx, now.Add(-s.config.NodeRetention))+ if err != nil {+ logger.Errorf("Failed to delete expired node records: %v", err)+ return+ }+ if deleted > 0 {+ nodesDeletedTotal.Add(float64(deleted))+ logger.Infof("Deleted %d node records whose session expired more than %s ago", deleted, s.config.NodeRetention)+ }+}diff --git a/internal/controlplane/node_gc_test.go b/internal/controlplane/node_gc_test.gonew file mode 100644index 00000000..6cb1a961--- /dev/null+++ b/internal/controlplane/node_gc_test.go@@ -0,0 +1,109 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package controlplane++import (+ "context"+ "testing"+ "time"++ "github.com/google/sam/api"+ "github.com/google/sam/internal/storage"+ dto "github.com/prometheus/client_model/go"+)++func TestGCExpiredNodesAppliesRetention(t *testing.T) {+ store := newMetricsTestStore(t)+ ctx := context.Background()+ now := time.Now()++ enroll := func(id string, expiresAt time.Time) {+ t.Helper()+ if err := store.EnrollNode(ctx, &storage.EnrolledNode{+ PeerID: id, PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode,+ EnrollmentType: "OIDC", EnrolledAt: now.Add(-90 * 24 * time.Hour), ExpiresAt: expiresAt,+ }); err != nil {+ t.Fatalf("enroll %s: %v", id, err)+ }+ }+ // Retention is 7d: only a session that lapsed more than a week ago goes.+ enroll("lapsed-8d", now.Add(-8*24*time.Hour))+ enroll("lapsed-6d", now.Add(-6*24*time.Hour))+ enroll("admitted", now.Add(24*time.Hour))++ srv, err := NewServer(Options{+ DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: "https://issuer",+ NodeRetention: 7 * 24 * time.Hour,+ }, store)+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = srv.Close() }()++ var before dto.Metric+ _ = nodesDeletedTotal.Write(&before)++ srv.gcExpiredNodes(now)++ nodes, err := store.ListNodes(ctx)+ if err != nil {+ t.Fatal(err)+ }+ if len(nodes) != 2 {+ t.Fatalf("got %d nodes after sweep, want 2: %+v", len(nodes), nodes)+ }+ for _, n := range nodes {+ if n.PeerID == "lapsed-8d" {+ t.Error("lapsed-8d survived a 7d retention")+ }+ }++ var after dto.Metric+ _ = nodesDeletedTotal.Write(&after)+ if got := after.GetCounter().GetValue() - before.GetCounter().GetValue(); got != 1 {+ t.Errorf("sam_control_plane_nodes_deleted_total advanced by %v, want 1", got)+ }+}++func TestNodeGCLoopIsOffWithoutRetention(t *testing.T) {+ store := newMetricsTestStore(t)+ ctx := context.Background()+ now := time.Now()+ if err := store.EnrollNode(ctx, &storage.EnrolledNode{+ PeerID: "lapsed", PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode,+ EnrollmentType: "OIDC", EnrolledAt: now.Add(-2 * time.Hour), ExpiresAt: now.Add(-time.Hour),+ }); err != nil {+ t.Fatal(err)+ }++ srv, err := NewServer(Options{DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: "https://issuer"}, store)+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = srv.Close() }()++ // The loop returns immediately with retention 0, and a direct sweep is a+ // no-op too: zero means keep forever, not "expired as of now".+ srv.wg.Add(1)+ srv.runNodeGCLoop()+ srv.gcExpiredNodes(now)+ nodes, err := store.ListNodes(ctx)+ if err != nil {+ t.Fatal(err)+ }+ if len(nodes) != 1 {+ t.Fatalf("retention 0 deleted rows: %d left", len(nodes))+ }+}diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex d440f6b0..54023db8 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -213,6 +213,9 @@ func (s *Server) Init() error {s.wg.Add(1)go s.runKeyRotationLoop()+ s.wg.Add(1)+ go s.runNodeGCLoop()+return nil}diff --git a/internal/storage/round_trip_test.go b/internal/storage/round_trip_test.goindex 83b0d033..3dfdf67b 100644--- a/internal/storage/round_trip_test.go+++ b/internal/storage/round_trip_test.go@@ -243,6 +243,61 @@ func TestReEnrollmentKeepsAnExistingBan(t *testing.T) {}}+// Expiry-based deletion must never take a ban with it, nor touch a node that+// never had a session bound: the first is how a node is kept out, the second+// has no expiry to have lapsed.+func TestDeleteExpiredNodesSparesBannedAndUnbounded(t *testing.T) {+ store := newTestStore(t)+ ctx := context.Background()+ now := time.Now()++ enroll := func(id string, expiresAt time.Time) {+ t.Helper()+ if err := store.EnrollNode(ctx, &EnrolledNode{+ PeerID: id, PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode,+ EnrollmentType: "OIDC", EnrolledAt: now.Add(-48 * time.Hour), ExpiresAt: expiresAt,+ }); err != nil {+ t.Fatalf("enroll %s: %v", id, err)+ }+ }+ enroll("lapsed-long-ago", now.Add(-24*time.Hour))+ enroll("lapsed-just-now", now.Add(-time.Minute))+ enroll("still-admitted", now.Add(time.Hour))+ enroll("lapsed-but-banned", now.Add(-24*time.Hour))+ enroll("never-bounded", time.Time{})+ if err := store.SetNodeBanned(ctx, "lapsed-but-banned", true); err != nil {+ t.Fatal(err)+ }++ deleted, err := store.DeleteExpiredNodes(ctx, now.Add(-time.Hour))+ if err != nil {+ t.Fatalf("DeleteExpiredNodes: %v", err)+ }+ if deleted != 1 {+ t.Errorf("deleted %d rows, want 1", deleted)+ }++ remaining, err := store.ListNodes(ctx)+ if err != nil {+ t.Fatal(err)+ }+ got := map[string]bool{}+ for _, n := range remaining {+ got[n.PeerID] = true+ }+ for _, want := range []string{"lapsed-just-now", "still-admitted", "lapsed-but-banned", "never-bounded"} {+ if !got[want] {+ t.Errorf("%s was deleted", want)+ }+ }+ if got["lapsed-long-ago"] {+ t.Error("lapsed-long-ago survived")+ }+ if banned, _ := store.IsNodeBanned(ctx, "lapsed-but-banned"); !banned {+ t.Error("the ban did not survive the sweep")+ }+}+func TestBootstrapTokenRoundTripsEveryField(t *testing.T) {store := newTestStore(t)ctx := context.Background()diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.goindex d13051c2..6c3f01fa 100644--- a/internal/storage/sql_store.go+++ b/internal/storage/sql_store.go@@ -892,6 +892,17 @@ func (s *SQLStore) IsIdentityBanned(ctx context.Context, identity string) (bool,return true, nil}+// DeleteExpiredNodes implements Store.+func (s *SQLStore) DeleteExpiredNodes(ctx context.Context, before time.Time) (int64, error) {+ // A zero ExpiresAt round-trips as a negative UnixMilli, hence > 0.+ query := s.rebind(`DELETE FROM nodes WHERE banned = ? AND expires_at > 0 AND expires_at < ?`)+ res, err := s.db.ExecContext(ctx, query, false, before.UnixMilli())+ if err != nil {+ return 0, err+ }+ return res.RowsAffected()+}+// ListBannedPeerIDs implements Store.func (s *SQLStore) ListBannedPeerIDs(ctx context.Context) ([]string, error) {query := s.rebind(`SELECT peer_id FROM nodes WHERE banned = ?`)diff --git a/internal/storage/storage.go b/internal/storage/storage.goindex 46554fdc..48103a99 100644--- a/internal/storage/storage.go+++ b/internal/storage/storage.go@@ -287,6 +287,14 @@ type Store interface {// ListNodes retrieves all enrolled nodes.ListNodes(ctx context.Context) ([]EnrolledNode, error)+ // DeleteExpiredNodes removes enrolled nodes whose session expired before+ // the given time and that are not banned, returning how many went. A+ // lapsed session already refuses every refresh, so the row only records+ // that the node once existed; a ban, by contrast, is the row, and must+ // outlive the session it was placed on. Nodes with no session bound+ // (zero ExpiresAt) are never removed.+ DeleteExpiredNodes(ctx context.Context, before time.Time) (int64, error)+// ListBootstrapTokens retrieves all bootstrap tokens.ListBootstrapTokens(ctx context.Context) ([]BootstrapToken, error)diff --git a/site/content/docs/user/control-plane-configuration.md b/site/content/docs/user/control-plane-configuration.mdindex 7fd625ed..551d9aac 100644--- a/site/content/docs/user/control-plane-configuration.md+++ b/site/content/docs/user/control-plane-configuration.md@@ -28,6 +28,9 @@ The Control Plane is responsible for bridging user identities from trusted OIDC| `--key-rotation-interval` | *None* | `24h` | Key rotation interval (e.g. `24h`). `0s` disables rotation. || `--key-grace-period` | *None* | `1h` | How long a rotated-out signing key stays accepted. Once it is retired, Biscuits it signed can no longer be verified or refreshed; see [Signing-Key Retirement and Recovery](#signing-key-retirement-and-recovery). || `--lease-duration` | *None* | `15m` | Router lease registration TTL. |+| `--node-retention` | *None* | `720h` (30 days) | How long an enrolled node's record is kept after its session expired before it is deleted. Every restart without a persistent data directory enrolls a fresh identity, so without this the node table only grows. Banned nodes are always kept. `0s` keeps every record forever. |++The Control Plane serves Prometheus metrics on `/metrics` alongside the API: mesh size (`sam_control_plane_mesh_connected_peers`, `sam_control_plane_routers_active`, `sam_control_plane_enrolled_nodes{role,state}`), per-route request counts and latencies, and the Go runtime.---@@ -49,6 +52,7 @@ The Router is a dedicated GossipSub helper that maintains stable network address| `--keys-sync-interval` | `5m` | Key synchronization polling interval. `GET /keys` returns the valid key set signed by every key in it; the router (and nodes, at start-up) accept the set only if one of those signatures verifies under a key they already trust, so the first key always comes from enrollment and a rotation is learned from the retiring key. || `--lease-renew-interval` | `300s` | Lease renewal registration interval. || `--allow-loopback` | `false` | Allow loopback and link-local addresses for discovery (development only). |+| `--metrics-addr` | *None* | Serve Prometheus `/metrics`, `/healthz` and `/readyz` on this plain HTTP address (e.g. `0.0.0.0:9090`). Off by default: nothing on it is authenticated, so keep it off the libp2p ports and inside the cluster. `/readyz` turns `200` once the router is enrolled and its libp2p host is online. The same flag exists on `sam-node run`. |---