Skip to content

Agent Mesh

controlplane: atomic bans, indexed scans, bounded revocation cache

Antonio Ojea34ddecdbebf8
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index df8389f9..ab5dafb1 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -3247,20 +3247,15 @@ func (s *Server) banNode(ctx context.Context, node *storage.EnrolledNode) error
// is shed with a new keypair, and an unban that lifts only the peer id leaves
// the human locked out of /register.
func SetNodeBan(ctx context.Context, store storage.Store, node *storage.EnrolledNode, banned bool) error {
- if err := store.SetNodeBanned(ctx, node.PeerID, banned); err != nil {
- return err
- }
- if node.ClaimsJSON == "" {
- return nil
- }
- var claims jwt.MapClaims
- if err := json.Unmarshal([]byte(node.ClaimsJSON), &claims); err != nil {
- return fmt.Errorf("stored claims for %s are unreadable: %w", node.PeerID, err)
- }
- if key := oidcIdentityKey(claims); key != "" {
- return store.SetIdentityBanned(ctx, key, banned)
+ var identity string
+ if node.ClaimsJSON != "" {
+ var claims jwt.MapClaims
+ if err := json.Unmarshal([]byte(node.ClaimsJSON), &claims); err != nil {
+ return fmt.Errorf("stored claims for %s are unreadable: %w", node.PeerID, err)
+ }
+ identity = oidcIdentityKey(claims)
}
- return nil
+ return store.SetNodeAndIdentityBanned(ctx, node.PeerID, identity, banned)
}
// allowedLabelPatterns collects the label grants of every role an identity
diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.go
index b6cd076a..c9833da9 100644
--- a/internal/controlplane/sts.go
+++ b/internal/controlplane/sts.go
@@ -429,7 +429,15 @@ func (s *Server) RevokeBiscuitID(revocationID string, expiry time.Time) {
if expiry.IsZero() {
expiry = time.Now().Add(s.config.BiscuitTTL)
}
+ now := time.Now()
s.revokedBiscuitsMu.Lock()
+ // Expired entries also go on read; dropping them here too keeps the map
+ // bounded when nothing polls /revocations.
+ for id, exp := range s.revokedBiscuits {
+ if !now.Before(exp) {
+ delete(s.revokedBiscuits, id)
+ }
+ }
s.revokedBiscuits[revocationID] = expiry
s.revokedBiscuitsMu.Unlock()
if s.store != nil {
diff --git a/internal/controlplane/sts_test.go b/internal/controlplane/sts_test.go
index 9a074d0c..840a8b8c 100644
--- a/internal/controlplane/sts_test.go
+++ b/internal/controlplane/sts_test.go
@@ -932,3 +932,25 @@ func TestOAuthAndControlPlaneHardening(t *testing.T) {
t.Fatalf("expected 415 for text/plain POST /policies, got %d", polResp.StatusCode)
}
}
+
+// A revocation that expired leaves the in-memory set on the next write, not
+// only when /revocations is read.
+func TestRevokeBiscuitIDPrunesExpiredEntries(t *testing.T) {
+ srv, store, _ := setupTestServer(t, "")
+ defer func() {
+ _ = srv.Close()
+ _ = store.Close()
+ }()
+
+ srv.RevokeBiscuitID("expired", time.Now().Add(-time.Minute))
+ srv.RevokeBiscuitID("live", time.Now().Add(time.Hour))
+
+ srv.revokedBiscuitsMu.RLock()
+ defer srv.revokedBiscuitsMu.RUnlock()
+ if _, ok := srv.revokedBiscuits["expired"]; ok {
+ t.Error("expired revocation still cached")
+ }
+ if _, ok := srv.revokedBiscuits["live"]; !ok {
+ t.Error("live revocation missing")
+ }
+}
diff --git a/internal/storage/migration_test.go b/internal/storage/migration_test.go
index fba61d46..45d670dd 100644
--- a/internal/storage/migration_test.go
+++ b/internal/storage/migration_test.go
@@ -18,6 +18,7 @@ import (
"context"
"database/sql"
"path/filepath"
+ "reflect"
"testing"
"time"
@@ -147,3 +148,36 @@ func TestMigrationConvertsSecondsToMillis(t *testing.T) {
t.Errorf("banned_at = %d, want %d", bannedAt, created.UnixMilli())
}
}
+
+// Migration 16 indexes the scans that run on every bootstrap and every GC
+// pass; the index list is the contract the planner relies on.
+func TestMigrationCreatesIndexes(t *testing.T) {
+ store, err := NewSQLStore("sqlite", filepath.Join(t.TempDir(), "cp.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+
+ rows, err := store.db.Query(`SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE 'idx_%' ORDER BY name`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = rows.Close() }()
+ var got []string
+ for rows.Next() {
+ var name string
+ if err := rows.Scan(&name); err != nil {
+ t.Fatal(err)
+ }
+ got = append(got, name)
+ }
+ want := []string{
+ "idx_enrollment_requests_created_at",
+ "idx_nodes_banned_expires_at",
+ "idx_revoked_biscuits_expires_at",
+ "idx_routers_expires_at",
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("indexes = %v, want %v", got, want)
+ }
+}
diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.go
index 31811905..913d886d 100644
--- a/internal/storage/sql_store.go
+++ b/internal/storage/sql_store.go
@@ -527,6 +527,21 @@ var migrations = []migration{
postgres: secondsToMillisMigration,
sqlite: secondsToMillisMigration,
},
+ {
+ // The scans that run on every bootstrap (/info's ban set), every
+ // lease lookup, every revocation pull and the node GC; nodes is the
+ // one table that grows with the mesh.
+ version: 16,
+ postgres: indexMigration,
+ sqlite: indexMigration,
+ },
+}
+
+var indexMigration = []string{
+ `CREATE INDEX IF NOT EXISTS idx_nodes_banned_expires_at ON nodes (banned, expires_at)`,
+ `CREATE INDEX IF NOT EXISTS idx_routers_expires_at ON routers (expires_at)`,
+ `CREATE INDEX IF NOT EXISTS idx_revoked_biscuits_expires_at ON revoked_biscuits (expires_at)`,
+ `CREATE INDEX IF NOT EXISTS idx_enrollment_requests_created_at ON enrollment_requests (created_at)`,
}
var secondsToMillisMigration = []string{
@@ -964,6 +979,38 @@ func (s *SQLStore) SetIdentityBanned(ctx context.Context, identity string, banne
return err
}
+// SetNodeAndIdentityBanned implements Store.
+func (s *SQLStore) SetNodeAndIdentityBanned(ctx context.Context, peerID, identity string, banned bool) error {
+ tx, err := s.db.BeginTx(ctx, nil)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = tx.Rollback() }()
+
+ res, err := tx.ExecContext(ctx, s.rebind(`UPDATE nodes SET banned = ? WHERE peer_id = ?`), banned, peerID)
+ if err != nil {
+ return err
+ }
+ n, err := res.RowsAffected()
+ if err != nil {
+ return err
+ }
+ if n == 0 {
+ return ErrNotFound
+ }
+ if identity != "" {
+ if banned {
+ _, err = tx.ExecContext(ctx, s.rebind(`INSERT INTO banned_identities (identity, banned_at) VALUES (?, ?) ON CONFLICT (identity) DO NOTHING`), identity, time.Now().UnixMilli())
+ } else {
+ _, err = tx.ExecContext(ctx, s.rebind(`DELETE FROM banned_identities WHERE identity = ?`), identity)
+ }
+ if err != nil {
+ return err
+ }
+ }
+ return tx.Commit()
+}
+
// IsIdentityBanned implements Store.
func (s *SQLStore) IsIdentityBanned(ctx context.Context, identity string) (bool, error) {
query := s.rebind(`SELECT 1 FROM banned_identities WHERE identity = ?`)
diff --git a/internal/storage/sql_store_test.go b/internal/storage/sql_store_test.go
index 87bd0ec4..e44f1f19 100644
--- a/internal/storage/sql_store_test.go
+++ b/internal/storage/sql_store_test.go
@@ -421,6 +421,48 @@ func TestIdentityBanOps(t *testing.T) {
}
}
+// A ban names the device and the identity behind it in one transaction: an
+// unknown peer rolls the identity ban back, so the two never disagree.
+func TestSetNodeAndIdentityBannedIsAtomic(t *testing.T) {
+ store := newTestStore(t)
+ defer func() { _ = store.Close() }()
+ ctx := context.Background()
+ const identity = "http://issuer.example|atomic-sub"
+
+ err := store.SetNodeAndIdentityBanned(ctx, "12D3KooWNoSuchNode", identity, true)
+ if err != ErrNotFound {
+ t.Fatalf("banning an unknown node: got %v, want ErrNotFound", err)
+ }
+ if banned, _ := store.IsIdentityBanned(ctx, identity); banned {
+ t.Fatal("identity ban survived the rolled-back node ban")
+ }
+
+ node := &EnrolledNode{PeerID: "12D3KooWAtomicNode", PublicKey: []byte("pk"), Biscuit: []byte("b"), Role: api.RoleNode, EnrollmentType: "oidc", EnrolledAt: time.Now(), ExpiresAt: time.Now().Add(time.Hour)}
+ if err := store.EnrollNode(ctx, node); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SetNodeAndIdentityBanned(ctx, node.PeerID, identity, true); err != nil {
+ t.Fatalf("ban: %v", err)
+ }
+ nodeBanned, _ := store.IsNodeBanned(ctx, node.PeerID)
+ identityBanned, _ := store.IsIdentityBanned(ctx, identity)
+ if !nodeBanned || !identityBanned {
+ t.Fatalf("after ban: node=%v identity=%v, want both true", nodeBanned, identityBanned)
+ }
+ if err := store.SetNodeAndIdentityBanned(ctx, node.PeerID, identity, false); err != nil {
+ t.Fatalf("unban: %v", err)
+ }
+ nodeBanned, _ = store.IsNodeBanned(ctx, node.PeerID)
+ identityBanned, _ = store.IsIdentityBanned(ctx, identity)
+ if nodeBanned || identityBanned {
+ t.Fatalf("after unban: node=%v identity=%v, want both false", nodeBanned, identityBanned)
+ }
+ // A bootstrap-enrolled node has no identity to ban.
+ if err := store.SetNodeAndIdentityBanned(ctx, node.PeerID, "", true); err != nil {
+ t.Fatalf("ban without identity: %v", err)
+ }
+}
+
func TestRouterLeaseOps(t *testing.T) {
store := newTestStore(t)
defer func() { _ = store.Close() }()
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index 0221fdec..aaae9290 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -231,6 +231,12 @@ type Store interface {
// this is what makes a ban survive keypair regeneration.
SetIdentityBanned(ctx context.Context, identity string, banned bool) error
+ // SetNodeAndIdentityBanned is SetNodeBanned and SetIdentityBanned in one
+ // transaction, so a failure cannot leave the device banned while its
+ // identity may still enroll a new one, or the other way round. An empty
+ // identity bans the node alone.
+ SetNodeAndIdentityBanned(ctx context.Context, peerID, identity string, banned bool) error
+
// IsIdentityBanned checks if an identity is currently banned.
IsIdentityBanned(ctx context.Context, identity string) (bool, error)