Skip to content

Agent Mesh

controlplane: delete node records that outlived their session

Antonio Ojeaa4faefbd435c
diff --git a/cmd/sam-control-plane/main.go b/cmd/sam-control-plane/main.go
index 62b281d9..f9c7ae7b 100644
--- a/cmd/sam-control-plane/main.go
+++ b/cmd/sam-control-plane/main.go
@@ -45,6 +45,7 @@ var (
leaseDuration time.Duration
biscuitTTL time.Duration
oidcSessionTTL time.Duration
+ nodeRetention time.Duration
adminTokenPath string
insecureSkipTLSVerify bool
logLevel string
@@ -126,6 +127,7 @@ func main() {
BiscuitTimeout: 10 * time.Second,
BiscuitTTL: biscuitTTL,
OIDCSessionTTL: oidcSessionTTL,
+ NodeRetention: nodeRetention,
AdminToken: adminToken,
AutoApproveEnrollment: autoApproveEnrollment,
}
@@ -161,6 +163,7 @@ func main() {
rootCmd.Flags().DurationVar(&leaseDuration, "lease-duration", 15*time.Minute, "Router lease registration TTL.")
rootCmd.Flags().DurationVar(&biscuitTTL, "biscuit-ttl", api.BiscuitTokenTTL, "Lifespan minted into every issued Biscuit's expiration fact. Capped to the OIDC token's own expiry when shorter.")
rootCmd.Flags().DurationVar(&oidcSessionTTL, "oidc-session-ttl", api.OIDCSessionTTL, "How long an OIDC enrollment stays refreshable before the identity must re-authenticate with the OIDC provider. Shorter values keep the provider authoritative for offboarding at the cost of more frequent interactive re-enrollment.")
+ rootCmd.Flags().DurationVar(&nodeRetention, "node-retention", controlplane.DefaultNodeRetention, "How long an enrolled node's record is kept after its session expired before it is deleted. Banned nodes are always kept. 0 keeps every record forever.")
rootCmd.Flags().StringVar(&adminTokenPath, "admin-token-path", "", "Path to file containing the token for authenticating policy REST API requests (or env SAM_ADMIN_TOKEN)")
rootCmd.Flags().BoolVar(&insecureSkipTLSVerify, "insecure-skip-tls-verify", false, "Skip TLS verification for OIDC providers")
rootCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")
diff --git a/internal/controlplane/config.go b/internal/controlplane/config.go
index b090a47a..20ad86bf 100644
--- a/internal/controlplane/config.go
+++ b/internal/controlplane/config.go
@@ -36,8 +36,13 @@ type Options struct {
BiscuitTimeout time.Duration
BiscuitTTL time.Duration // Lifespan minted into every issued Biscuit's expiration() fact; defaults to api.BiscuitTokenTTL
OIDCSessionTTL time.Duration // How long an OIDC enrollment stays refreshable before the identity must re-authenticate interactively; defaults to api.OIDCSessionTTL
- AdminToken string // Optional: administrative bearer token for protecting policy and enrollment queue REST APIs
- AutoApproveEnrollment bool // If true, valid bootstrap token enrollment requests are immediately approved without administrative manual gate
+ // NodeRetention is how long an enrolled node's row is kept after its
+ // session expired before being deleted; 0 keeps rows forever. Every
+ // pod restart without a persistent data dir enrolls a fresh identity,
+ // so without this the nodes table only ever grows.
+ NodeRetention time.Duration
+ AdminToken string // Optional: administrative bearer token for protecting policy and enrollment queue REST APIs
+ AutoApproveEnrollment bool // If true, valid bootstrap token enrollment requests are immediately approved without administrative manual gate
}
// Default sets default values for control plane options.
diff --git a/internal/controlplane/node_gc.go b/internal/controlplane/node_gc.go
new file mode 100644
index 00000000..07ff5844
--- /dev/null
+++ b/internal/controlplane/node_gc.go
@@ -0,0 +1,83 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+ "context"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+ // DefaultNodeRetention keeps a lapsed enrollment for a month past its
+ // session, long enough to be looked up when investigating an incident,
+ // and is what --node-retention defaults to.
+ DefaultNodeRetention = 30 * 24 * time.Hour
+
+ // nodeGCInterval is how often the sweep runs. Rows lapse on a scale of
+ // days, so nothing is gained by sweeping more often than hourly.
+ nodeGCInterval = time.Hour
+)
+
+var nodesDeletedTotal = promauto.NewCounter(prometheus.CounterOpts{
+ Name: "sam_control_plane_nodes_deleted_total",
+ Help: "Enrolled node records removed after outliving their session by the retention period",
+})
+
+// runNodeGCLoop sweeps once at start and then hourly. Replicas each sweep
+// against the shared database; the delete is idempotent, so the race is
+// harmless and not worth a claim.
+func (s *Server) runNodeGCLoop() {
+ defer s.wg.Done()
+ if s.config.NodeRetention <= 0 {
+ return
+ }
+
+ s.gcExpiredNodes(time.Now())
+
+ ticker := time.NewTicker(nodeGCInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ s.gcExpiredNodes(time.Now())
+ case <-s.ctx.Done():
+ return
+ }
+ }
+}
+
+// gcExpiredNodes deletes every unbanned node whose session expired more than
+// NodeRetention before now. A retention of zero means keep forever, and that
+// holds here too, not only in the loop that decides whether to tick.
+func (s *Server) gcExpiredNodes(now time.Time) {
+ if s.config.NodeRetention <= 0 {
+ return
+ }
+ ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
+ defer cancel()
+
+ deleted, err := s.store.DeleteExpiredNodes(ctx, now.Add(-s.config.NodeRetention))
+ if err != nil {
+ logger.Errorf("Failed to delete expired node records: %v", err)
+ return
+ }
+ if deleted > 0 {
+ nodesDeletedTotal.Add(float64(deleted))
+ logger.Infof("Deleted %d node records whose session expired more than %s ago", deleted, s.config.NodeRetention)
+ }
+}
diff --git a/internal/controlplane/node_gc_test.go b/internal/controlplane/node_gc_test.go
new file mode 100644
index 00000000..6cb1a961
--- /dev/null
+++ b/internal/controlplane/node_gc_test.go
@@ -0,0 +1,109 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/google/sam/api"
+ "github.com/google/sam/internal/storage"
+ dto "github.com/prometheus/client_model/go"
+)
+
+func TestGCExpiredNodesAppliesRetention(t *testing.T) {
+ store := newMetricsTestStore(t)
+ ctx := context.Background()
+ now := time.Now()
+
+ enroll := func(id string, expiresAt time.Time) {
+ t.Helper()
+ if err := store.EnrollNode(ctx, &storage.EnrolledNode{
+ PeerID: id, PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode,
+ EnrollmentType: "OIDC", EnrolledAt: now.Add(-90 * 24 * time.Hour), ExpiresAt: expiresAt,
+ }); err != nil {
+ t.Fatalf("enroll %s: %v", id, err)
+ }
+ }
+ // Retention is 7d: only a session that lapsed more than a week ago goes.
+ enroll("lapsed-8d", now.Add(-8*24*time.Hour))
+ enroll("lapsed-6d", now.Add(-6*24*time.Hour))
+ enroll("admitted", now.Add(24*time.Hour))
+
+ srv, err := NewServer(Options{
+ DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: "https://issuer",
+ NodeRetention: 7 * 24 * time.Hour,
+ }, store)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = srv.Close() }()
+
+ var before dto.Metric
+ _ = nodesDeletedTotal.Write(&before)
+
+ srv.gcExpiredNodes(now)
+
+ nodes, err := store.ListNodes(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(nodes) != 2 {
+ t.Fatalf("got %d nodes after sweep, want 2: %+v", len(nodes), nodes)
+ }
+ for _, n := range nodes {
+ if n.PeerID == "lapsed-8d" {
+ t.Error("lapsed-8d survived a 7d retention")
+ }
+ }
+
+ var after dto.Metric
+ _ = nodesDeletedTotal.Write(&after)
+ if got := after.GetCounter().GetValue() - before.GetCounter().GetValue(); got != 1 {
+ t.Errorf("sam_control_plane_nodes_deleted_total advanced by %v, want 1", got)
+ }
+}
+
+func TestNodeGCLoopIsOffWithoutRetention(t *testing.T) {
+ store := newMetricsTestStore(t)
+ ctx := context.Background()
+ now := time.Now()
+ if err := store.EnrollNode(ctx, &storage.EnrolledNode{
+ PeerID: "lapsed", PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode,
+ EnrollmentType: "OIDC", EnrolledAt: now.Add(-2 * time.Hour), ExpiresAt: now.Add(-time.Hour),
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ srv, err := NewServer(Options{DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: "https://issuer"}, store)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = srv.Close() }()
+
+ // The loop returns immediately with retention 0, and a direct sweep is a
+ // no-op too: zero means keep forever, not "expired as of now".
+ srv.wg.Add(1)
+ srv.runNodeGCLoop()
+ srv.gcExpiredNodes(now)
+ nodes, err := store.ListNodes(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(nodes) != 1 {
+ t.Fatalf("retention 0 deleted rows: %d left", len(nodes))
+ }
+}
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index d440f6b0..54023db8 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -213,6 +213,9 @@ func (s *Server) Init() error {
s.wg.Add(1)
go s.runKeyRotationLoop()
+ s.wg.Add(1)
+ go s.runNodeGCLoop()
+
return nil
}
diff --git a/internal/storage/round_trip_test.go b/internal/storage/round_trip_test.go
index 83b0d033..3dfdf67b 100644
--- a/internal/storage/round_trip_test.go
+++ b/internal/storage/round_trip_test.go
@@ -243,6 +243,61 @@ func TestReEnrollmentKeepsAnExistingBan(t *testing.T) {
}
}
+// Expiry-based deletion must never take a ban with it, nor touch a node that
+// never had a session bound: the first is how a node is kept out, the second
+// has no expiry to have lapsed.
+func TestDeleteExpiredNodesSparesBannedAndUnbounded(t *testing.T) {
+ store := newTestStore(t)
+ ctx := context.Background()
+ now := time.Now()
+
+ enroll := func(id string, expiresAt time.Time) {
+ t.Helper()
+ if err := store.EnrollNode(ctx, &EnrolledNode{
+ PeerID: id, PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode,
+ EnrollmentType: "OIDC", EnrolledAt: now.Add(-48 * time.Hour), ExpiresAt: expiresAt,
+ }); err != nil {
+ t.Fatalf("enroll %s: %v", id, err)
+ }
+ }
+ enroll("lapsed-long-ago", now.Add(-24*time.Hour))
+ enroll("lapsed-just-now", now.Add(-time.Minute))
+ enroll("still-admitted", now.Add(time.Hour))
+ enroll("lapsed-but-banned", now.Add(-24*time.Hour))
+ enroll("never-bounded", time.Time{})
+ if err := store.SetNodeBanned(ctx, "lapsed-but-banned", true); err != nil {
+ t.Fatal(err)
+ }
+
+ deleted, err := store.DeleteExpiredNodes(ctx, now.Add(-time.Hour))
+ if err != nil {
+ t.Fatalf("DeleteExpiredNodes: %v", err)
+ }
+ if deleted != 1 {
+ t.Errorf("deleted %d rows, want 1", deleted)
+ }
+
+ remaining, err := store.ListNodes(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := map[string]bool{}
+ for _, n := range remaining {
+ got[n.PeerID] = true
+ }
+ for _, want := range []string{"lapsed-just-now", "still-admitted", "lapsed-but-banned", "never-bounded"} {
+ if !got[want] {
+ t.Errorf("%s was deleted", want)
+ }
+ }
+ if got["lapsed-long-ago"] {
+ t.Error("lapsed-long-ago survived")
+ }
+ if banned, _ := store.IsNodeBanned(ctx, "lapsed-but-banned"); !banned {
+ t.Error("the ban did not survive the sweep")
+ }
+}
+
func TestBootstrapTokenRoundTripsEveryField(t *testing.T) {
store := newTestStore(t)
ctx := context.Background()
diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.go
index d13051c2..6c3f01fa 100644
--- a/internal/storage/sql_store.go
+++ b/internal/storage/sql_store.go
@@ -892,6 +892,17 @@ func (s *SQLStore) IsIdentityBanned(ctx context.Context, identity string) (bool,
return true, nil
}
+// DeleteExpiredNodes implements Store.
+func (s *SQLStore) DeleteExpiredNodes(ctx context.Context, before time.Time) (int64, error) {
+ // A zero ExpiresAt round-trips as a negative UnixMilli, hence > 0.
+ query := s.rebind(`DELETE FROM nodes WHERE banned = ? AND expires_at > 0 AND expires_at < ?`)
+ res, err := s.db.ExecContext(ctx, query, false, before.UnixMilli())
+ if err != nil {
+ return 0, err
+ }
+ return res.RowsAffected()
+}
+
// ListBannedPeerIDs implements Store.
func (s *SQLStore) ListBannedPeerIDs(ctx context.Context) ([]string, error) {
query := s.rebind(`SELECT peer_id FROM nodes WHERE banned = ?`)
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index 46554fdc..48103a99 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -287,6 +287,14 @@ type Store interface {
// ListNodes retrieves all enrolled nodes.
ListNodes(ctx context.Context) ([]EnrolledNode, error)
+ // DeleteExpiredNodes removes enrolled nodes whose session expired before
+ // the given time and that are not banned, returning how many went. A
+ // lapsed session already refuses every refresh, so the row only records
+ // that the node once existed; a ban, by contrast, is the row, and must
+ // outlive the session it was placed on. Nodes with no session bound
+ // (zero ExpiresAt) are never removed.
+ DeleteExpiredNodes(ctx context.Context, before time.Time) (int64, error)
+
// ListBootstrapTokens retrieves all bootstrap tokens.
ListBootstrapTokens(ctx context.Context) ([]BootstrapToken, error)
diff --git a/site/content/docs/user/control-plane-configuration.md b/site/content/docs/user/control-plane-configuration.md
index 7fd625ed..551d9aac 100644
--- a/site/content/docs/user/control-plane-configuration.md
+++ b/site/content/docs/user/control-plane-configuration.md
@@ -28,6 +28,9 @@ The Control Plane is responsible for bridging user identities from trusted OIDC
| `--key-rotation-interval` | *None* | `24h` | Key rotation interval (e.g. `24h`). `0s` disables rotation. |
| `--key-grace-period` | *None* | `1h` | How long a rotated-out signing key stays accepted. Once it is retired, Biscuits it signed can no longer be verified or refreshed; see [Signing-Key Retirement and Recovery](#signing-key-retirement-and-recovery). |
| `--lease-duration` | *None* | `15m` | Router lease registration TTL. |
+| `--node-retention` | *None* | `720h` (30 days) | How long an enrolled node's record is kept after its session expired before it is deleted. Every restart without a persistent data directory enrolls a fresh identity, so without this the node table only grows. Banned nodes are always kept. `0s` keeps every record forever. |
+
+The Control Plane serves Prometheus metrics on `/metrics` alongside the API: mesh size (`sam_control_plane_mesh_connected_peers`, `sam_control_plane_routers_active`, `sam_control_plane_enrolled_nodes{role,state}`), per-route request counts and latencies, and the Go runtime.
---
@@ -49,6 +52,7 @@ The Router is a dedicated GossipSub helper that maintains stable network address
| `--keys-sync-interval` | `5m` | Key synchronization polling interval. `GET /keys` returns the valid key set signed by every key in it; the router (and nodes, at start-up) accept the set only if one of those signatures verifies under a key they already trust, so the first key always comes from enrollment and a rotation is learned from the retiring key. |
| `--lease-renew-interval` | `300s` | Lease renewal registration interval. |
| `--allow-loopback` | `false` | Allow loopback and link-local addresses for discovery (development only). |
+| `--metrics-addr` | *None* | Serve Prometheus `/metrics`, `/healthz` and `/readyz` on this plain HTTP address (e.g. `0.0.0.0:9090`). Off by default: nothing on it is authenticated, so keep it off the libp2p ports and inside the cluster. `/readyz` turns `200` once the router is enrolled and its libp2p host is online. The same flag exists on `sam-node run`. |
---