Skip to content

Agent Mesh

controlplane: --node-retention must be positive; no record is kept forever

Antonio Ojeaa84043a25956
diff --git a/cmd/agentmesh-control-plane/main.go b/cmd/agentmesh-control-plane/main.go
index 405cd496..ebf3f158 100644
--- a/cmd/agentmesh-control-plane/main.go
+++ b/cmd/agentmesh-control-plane/main.go
@@ -98,6 +98,10 @@ func main() {
if stsIssuerURL == "" {
stsIssuerURL = strings.TrimSpace(os.Getenv("AGENTMESH_STS_ISSUER_URL"))
}
+ // Options reads 0 as unset; the flag's 0 is a value the operator typed.
+ if nodeRetention <= 0 {
+ logger.Fatalf("--node-retention must be positive, got %s: node records are never kept forever", nodeRetention)
+ }
var auds []string
for _, aud := range strings.Split(allowedAudiencesFlag, ",") {
@@ -121,12 +125,6 @@ func main() {
}
}()
- // Options reads 0 as the default, so the flag's 0 (keep forever)
- // travels as a negative retention.
- if nodeRetention == 0 {
- nodeRetention = -1
- }
-
opts := controlplane.Options{
ListenAddr: bindAddress,
DriverName: dbDriver,
@@ -208,7 +206,7 @@ func main() {
rootCmd.Flags().DurationVar(&biscuitTTL, "biscuit-ttl", api.BiscuitTokenTTL, "Lifespan minted into every issued Biscuit's expiration fact. Capped to the OIDC token's own expiry when shorter.")
rootCmd.Flags().DurationVar(&oidcSessionTTL, "oidc-session-ttl", api.OIDCSessionTTL, "How long an OIDC enrollment stays refreshable before the identity must re-authenticate with the OIDC provider. Shorter values keep the provider authoritative for offboarding at the cost of more frequent interactive re-enrollment.")
rootCmd.Flags().DurationVar(&workloadSessionTTL, "workload-session-ttl", controlplane.DefaultWorkloadSessionTTL, "How long a workload-issuer enrollment stays refreshable without presenting a fresh platform JWT on /refresh.")
- rootCmd.Flags().DurationVar(&nodeRetention, "node-retention", controlplane.DefaultNodeRetention, "How long an enrolled node's record is kept after its session expired before it is deleted. Banned nodes are always kept. 0 keeps every record forever.")
+ rootCmd.Flags().DurationVar(&nodeRetention, "node-retention", controlplane.DefaultNodeRetention, "How long an enrolled node's record is kept after its session expired before it is deleted. Banned nodes are always kept. Must be positive: records are never kept forever.")
rootCmd.Flags().DurationVar(&meshReconnectInterval, "mesh-reconnect-interval", controlplane.DefaultMeshReconnectInterval, "How often the event publisher re-reads the router leases and dials any router it is not connected to.")
rootCmd.Flags().StringVar(&adminTokenPath, "admin-token-path", "", "Path to file containing the token for authenticating policy REST API requests (or env AGENTMESH_ADMIN_TOKEN)")
rootCmd.Flags().StringVar(&stsIssuerURL, "sts-issuer-url", "", "Canonical external URL of this control plane for OIDC/STS issuer and discovery metadata (or env AGENTMESH_STS_ISSUER_URL)")
diff --git a/cmd/agentmesh-one/main.go b/cmd/agentmesh-one/main.go
index 0e93a76e..9fade107 100644
--- a/cmd/agentmesh-one/main.go
+++ b/cmd/agentmesh-one/main.go
@@ -231,6 +231,7 @@ func main() {
rootCmd.Flags().DurationVar(&cpTunables.KeyGracePeriod, "control-plane-key-grace-period", 0, "How long rotated-out keys stay valid for verification (0 keeps the component default)")
rootCmd.Flags().DurationVar(&cpTunables.BiscuitTTL, "control-plane-biscuit-ttl", 0, "Lifespan minted into issued biscuits (0 keeps the component default)")
rootCmd.Flags().DurationVar(&cpTunables.WorkloadSessionTTL, "control-plane-workload-session-ttl", 0, "How long a workload-issuer enrollment stays refreshable without presenting a fresh platform JWT on /refresh (0 keeps the component default)")
+ rootCmd.Flags().DurationVar(&cpTunables.NodeRetention, "control-plane-node-retention", 0, "How long a node record is kept after its session expired before it is deleted (0 keeps the component default)")
rootCmd.Flags().BoolVar(&cpTunables.ManualEnrollment, "control-plane-manual-enrollment", false, "Queue bootstrap enrollments for admin approval instead of auto-approving")
// Embedded router tunables.
diff --git a/internal/controlplane/config.go b/internal/controlplane/config.go
index a71618d8..2f403ab9 100644
--- a/internal/controlplane/config.go
+++ b/internal/controlplane/config.go
@@ -49,9 +49,9 @@ type Options struct {
OIDCSessionTTL time.Duration // How long an OIDC enrollment stays refreshable before the identity must re-authenticate interactively; defaults to api.OIDCSessionTTL
WorkloadSessionTTL time.Duration // Session TTL for enrollments from a WorkloadIssuer; defaults to DefaultWorkloadSessionTTL (48h)
// NodeRetention is how long an enrolled node's row is kept after its
- // session expired before being deleted; 0 is DefaultNodeRetention and a
- // negative value keeps rows forever. Every pod restart without a
- // persistent data dir enrolls a fresh identity, so without this the
+ // session expired before being deleted; unset is DefaultNodeRetention,
+ // and Validate refuses a negative value. Every pod restart without a
+ // persistent data dir enrolls a fresh identity, so without a bound the
// nodes table only ever grows.
NodeRetention time.Duration
AdminToken string // Optional: administrative bearer token for protecting policy and enrollment queue REST APIs
@@ -153,5 +153,8 @@ func (o *Options) Validate() error {
if err := api.ValidateDirectoryParameters(o.DHTReplication, o.DHTReprovideInterval); err != nil {
return err
}
+ if o.NodeRetention <= 0 {
+ return fmt.Errorf("NodeRetention must be positive, got %s: node records are never kept forever", o.NodeRetention)
+ }
return nil
}
diff --git a/internal/controlplane/node_gc.go b/internal/controlplane/node_gc.go
index 972d7cd4..c7a5237a 100644
--- a/internal/controlplane/node_gc.go
+++ b/internal/controlplane/node_gc.go
@@ -43,10 +43,6 @@ var nodesDeletedTotal = promauto.NewCounter(prometheus.CounterOpts{
// harmless and not worth a claim.
func (s *Server) runNodeGCLoop() {
defer s.wg.Done()
- if s.config.NodeRetention <= 0 {
- return
- }
-
s.gcExpiredNodes(time.Now())
ticker := time.NewTicker(nodeGCInterval)
@@ -62,12 +58,10 @@ func (s *Server) runNodeGCLoop() {
}
// gcExpiredNodes deletes every unbanned node whose session expired more than
-// NodeRetention before now. A negative retention means keep forever, and that
-// holds here too, not only in the loop that decides whether to tick.
+// NodeRetention before now. Options.Validate has refused a retention that is
+// not positive, so there is no "keep forever": an unbounded nodes table is
+// what slowed every console refresh and metrics scrape.
func (s *Server) gcExpiredNodes(now time.Time) {
- if s.config.NodeRetention <= 0 {
- return
- }
ctx, cancel := context.WithTimeout(s.ctx, 30*time.Second)
defer cancel()
diff --git a/internal/controlplane/node_gc_test.go b/internal/controlplane/node_gc_test.go
index c4ae136b..f15ca172 100644
--- a/internal/controlplane/node_gc_test.go
+++ b/internal/controlplane/node_gc_test.go
@@ -16,6 +16,7 @@ package controlplane
import (
"context"
+ "strings"
"testing"
"time"
@@ -77,34 +78,16 @@ func TestGCExpiredNodesAppliesRetention(t *testing.T) {
}
}
-func TestNodeGCLoopIsOffWithoutRetention(t *testing.T) {
+// A retention that is not positive is refused, not read as "keep forever":
+// an unbounded nodes table is what this GC exists to prevent. Unset is the
+// default (TestGCExpiredNodesDefaultsRetention).
+func TestNodeRetentionMustBePositive(t *testing.T) {
store := newMetricsTestStore(t)
- ctx := context.Background()
- now := time.Now()
- if err := store.EnrollNode(ctx, &storage.EnrolledNode{
- PeerID: "lapsed", PublicKey: []byte("pub"), Biscuit: []byte("b"), Role: api.RoleNode,
- EnrollmentType: "OIDC", EnrolledAt: now.Add(-2 * time.Hour), ExpiresAt: now.Add(-time.Hour),
- }); err != nil {
- t.Fatal(err)
- }
-
- srv, err := NewServer(Options{DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: "https://issuer", NodeRetention: -1}, store)
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = srv.Close() }()
-
- // The loop returns immediately with a negative retention, and a direct
- // sweep is a no-op too: it means keep forever, not "expired as of now".
- srv.wg.Add(1)
- srv.runNodeGCLoop()
- srv.gcExpiredNodes(now)
- nodes, err := store.ListNodes(ctx, storage.NodeFilter{})
- if err != nil {
- t.Fatal(err)
- }
- if len(nodes) != 1 {
- t.Fatalf("negative retention deleted rows: %d left", len(nodes))
+ for _, retention := range []time.Duration{-1, -time.Hour} {
+ _, err := NewServer(Options{DriverName: "sqlite", DataSourceName: "unused", OIDCIssuer: "https://issuer", NodeRetention: retention}, store)
+ if err == nil || !strings.Contains(err.Error(), "NodeRetention must be positive") {
+ t.Fatalf("NewServer with NodeRetention %s: err = %v, want a refusal naming NodeRetention", retention, err)
+ }
}
}
diff --git a/internal/standalone/standalone.go b/internal/standalone/standalone.go
index e5e5d105..f859edec 100644
--- a/internal/standalone/standalone.go
+++ b/internal/standalone/standalone.go
@@ -137,6 +137,9 @@ type ControlPlaneTunables struct {
// WorkloadSessionTTL is how long a workload-issuer enrollment stays
// refreshable without presenting a fresh platform JWT on /refresh.
WorkloadSessionTTL time.Duration
+ // NodeRetention is how long a node record is kept after its session
+ // expired before the control plane deletes it.
+ NodeRetention time.Duration
// ManualEnrollment queues bootstrap enrollments for admin approval
// instead of auto-approving them.
ManualEnrollment bool
@@ -290,6 +293,7 @@ func (s *Server) Start(ctx context.Context) (retErr error) {
KeyGracePeriod: s.opts.ControlPlane.KeyGracePeriod,
BiscuitTTL: s.opts.ControlPlane.BiscuitTTL,
WorkloadSessionTTL: s.opts.ControlPlane.WorkloadSessionTTL,
+ NodeRetention: s.opts.ControlPlane.NodeRetention,
BiscuitTimeout: 10 * time.Second,
AdminToken: s.adminToken,
AutoApproveEnrollment: !s.opts.ControlPlane.ManualEnrollment,
diff --git a/site/content/docs/reference/agentmesh-one.md b/site/content/docs/reference/agentmesh-one.md
index 0e3f572f..2755bb36 100644
--- a/site/content/docs/reference/agentmesh-one.md
+++ b/site/content/docs/reference/agentmesh-one.md
@@ -64,6 +64,7 @@ of zero leaves the default of that component unchanged.
| `--control-plane-key-grace-period` | `--key-grace-period` |
| `--control-plane-biscuit-ttl` | `--biscuit-ttl` |
| `--control-plane-workload-session-ttl` | `--workload-session-ttl` |
+| `--control-plane-node-retention` | `--node-retention` |
| `--control-plane-manual-enrollment` | The opposite of `--auto-approve-enrollment`: queue bootstrap enrollments for approval. |
| `--router-control-plane-sync-interval` | `agentmesh-router --control-plane-sync-interval` |
| `--router-low-watermark`, `--router-high-watermark` | `--low-watermark`, `--high-watermark` |
diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.md
index f78d1099..15f73a6c 100644
--- a/site/content/docs/reference/control-plane.md
+++ b/site/content/docs/reference/control-plane.md
@@ -39,7 +39,7 @@ agentmesh-control-plane admin unban --peer <id> lift a ban
| `--lease-duration` | `15m` | How long a router lease lasts without renewal. |
| `--dht-replication` | `3` | How many routers each record a member announces is placed on: the router it announced to keeps it, and places a copy on this many routers closest to the key. A record on three routers survives two of them down at once; each replica more is a copy more of every record. Members are not concerned: they ask the routers they hold. Published on `/info`; every router of the mesh runs on it. |
| `--dht-reprovide-interval` | `5m` | How often a member re-announces each service it publishes. A provider that stopped without unpublishing stays listed for three intervals, which is how long routers keep a record after its last announcement; the mesh pays its providers divided by the interval in announcements a minute. Published on `/info`. |
-| `--node-retention` | `720h` (30 days) | How long the record of an enrolled node is kept after its session expires. Banned nodes are kept forever. `0` keeps every record. |
+| `--node-retention` | `720h` (30 days) | How long the record of an enrolled node is kept after its session expires. Banned nodes are kept forever. Must be positive; records are never kept forever. |
| `--mesh-reconnect-interval` | `30s` | How often the event publisher re-reads the router leases and dials any router it is not connected to. On each router it runs the mesh handshake with a credential it mints for its own peer under the signing key, with the role `mesh:role:control-plane`, so the router holds its connection as a member's. |
| `--log-level` | `info` | `debug`, `info`, `warn`, `error`. |
| `--log-format` | `text` | `text` or `json`. |