Skip to content

Agent Mesh

fix(node): persist the trusted control plane key set

Antonio Ojeaa7cab1bdb807
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index 18138b21..3cc6b461 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -154,9 +154,7 @@ func (n *SamNode) processEnrollResponse(resp *http.Response) (*api.EnrollRespons
return nil, fmt.Errorf("failed to save mesh config: %v", err)
}
- n.keysMu.Lock()
- n.trustedKeys = append(n.trustedKeys, TrustedKey{Key: ed25519.PublicKey(enrollResp.ControlPlanePublicKey), ReceivedAt: time.Now()})
- n.keysMu.Unlock()
+ n.addTrustedKey(ed25519.PublicKey(enrollResp.ControlPlanePublicKey))
return &enrollResp, nil
}
@@ -335,9 +333,7 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, b
return fmt.Errorf("failed to save mesh config: %v", err)
}
- n.keysMu.Lock()
- n.trustedKeys = append(n.trustedKeys, TrustedKey{Key: ed25519.PublicKey(enrollResp.ControlPlanePublicKey), ReceivedAt: time.Now()})
- n.keysMu.Unlock()
+ n.addTrustedKey(ed25519.PublicKey(enrollResp.ControlPlanePublicKey))
// Connect and Auth to router after enrollment to join the mesh
if len(enrollResp.RouterAddresses) == 0 {
diff --git a/internal/node/node.go b/internal/node/node.go
index 0f3f7aea..99fbdc23 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -255,8 +255,15 @@ func NewSamNode(cfg Options) (*SamNode, error) {
}
var trustedKeys []TrustedKey
- if len(cfg.ControlPlanePubKey) > 0 {
- trustedKeys = []TrustedKey{{Key: cfg.ControlPlanePubKey, ReceivedAt: time.Now()}}
+ if cfg.Store != nil {
+ if stored, err := cfg.Store.LoadTrustedKeys(); err != nil {
+ logger.Warnf("Failed to load persisted trusted keys: %v", err)
+ } else {
+ trustedKeys = stored
+ }
+ }
+ if len(cfg.ControlPlanePubKey) > 0 && !containsTrustedKey(trustedKeys, cfg.ControlPlanePubKey) {
+ trustedKeys = append(trustedKeys, TrustedKey{Key: cfg.ControlPlanePubKey, ReceivedAt: time.Now()})
}
node := &SamNode{
@@ -1230,20 +1237,45 @@ func (n *SamNode) handleBannedEvent(event *api.MeshEvent) {
}
}
+func containsTrustedKey(keys []TrustedKey, key ed25519.PublicKey) bool {
+ for _, tk := range keys {
+ if bytes.Equal(tk.Key, key) {
+ return true
+ }
+ }
+ return false
+}
+
+// addTrustedKey appends a control plane public key to the trust set (no-op
+// on duplicates) and persists the updated set so it survives restarts.
+func (n *SamNode) addTrustedKey(key ed25519.PublicKey) {
+ n.keysMu.Lock()
+ if containsTrustedKey(n.trustedKeys, key) {
+ n.keysMu.Unlock()
+ return
+ }
+ n.trustedKeys = append(n.trustedKeys, TrustedKey{Key: key, ReceivedAt: time.Now()})
+ snapshot := append([]TrustedKey(nil), n.trustedKeys...)
+ n.keysMu.Unlock()
+ n.persistTrustedKeys(snapshot)
+}
+
+func (n *SamNode) persistTrustedKeys(keys []TrustedKey) {
+ if n.Store == nil {
+ return
+ }
+ if err := n.Store.SaveTrustedKeys(keys); err != nil {
+ logger.Errorf("Failed to persist trusted keys: %v", err)
+ }
+}
+
func (n *SamNode) handleKeyRotationEvent(event *api.MeshEvent) {
if len(event.NewPublicKey) != ed25519.PublicKeySize {
logger.Errorf("[Mesh Event] Key rotation failed: invalid public key size %d, expected %d", len(event.NewPublicKey), ed25519.PublicKeySize)
return
}
logger.Infow("[Mesh Event] key rotation received", "event", meshEventKeyRotation, "key", fmt.Sprintf("%x", event.NewPublicKey))
- n.keysMu.Lock()
- defer n.keysMu.Unlock()
- for _, tk := range n.trustedKeys {
- if bytes.Equal(tk.Key, event.NewPublicKey) {
- return // Ignore duplicate
- }
- }
- n.trustedKeys = append(n.trustedKeys, TrustedKey{Key: ed25519.PublicKey(event.NewPublicKey), ReceivedAt: time.Now()})
+ n.addTrustedKey(ed25519.PublicKey(event.NewPublicKey))
}
// pruneTrustedKeys drops keys older than gracePeriod but always keeps the
@@ -1281,8 +1313,14 @@ func (n *SamNode) startKeyPruning(ctx context.Context, gracePeriod time.Duration
case <-ticker.C:
logger.Info("[KeyPruning] Pruning expired keys...")
n.keysMu.Lock()
- n.trustedKeys = pruneTrustedKeys(n.trustedKeys, time.Now(), gracePeriod)
+ pruned := pruneTrustedKeys(n.trustedKeys, time.Now(), gracePeriod)
+ changed := len(pruned) != len(n.trustedKeys)
+ n.trustedKeys = pruned
+ snapshot := append([]TrustedKey(nil), pruned...)
n.keysMu.Unlock()
+ if changed {
+ n.persistTrustedKeys(snapshot)
+ }
case <-ctx.Done():
return
}
diff --git a/internal/node/node_test.go b/internal/node/node_test.go
index 44eb151d..b0177c07 100644
--- a/internal/node/node_test.go
+++ b/internal/node/node_test.go
@@ -145,6 +145,49 @@ func TestHandleKeyRotationEvent(t *testing.T) {
}
}
+// TestKeyRotationEventSurvivesRestart covers the offline/restart half of
+// #325: a key learned from a rotation event must outlive the process, or a
+// restarted node falls back to its stale enrollment-time key.
+func TestKeyRotationEventSurvivesRestart(t *testing.T) {
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+
+ enrollKey, _, _ := ed25519.GenerateKey(nil)
+ rotatedKey, _, _ := ed25519.GenerateKey(nil)
+
+ node := &SamNode{Store: store, trustedKeys: []TrustedKey{{Key: enrollKey, ReceivedAt: time.Now()}}}
+ node.handleKeyRotationEvent(&api.MeshEvent{
+ Type: api.MeshEvent_KEY_ROTATION,
+ NewPublicKey: rotatedKey,
+ Timestamp: time.Now().UnixMilli(),
+ })
+ // Duplicate event must not grow the persisted set
+ node.handleKeyRotationEvent(&api.MeshEvent{
+ Type: api.MeshEvent_KEY_ROTATION,
+ NewPublicKey: rotatedKey,
+ Timestamp: time.Now().UnixMilli(),
+ })
+
+ // "Restart": a fresh node built from the same store must trust the rotated key
+ priv, _, _ := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ restarted, err := NewSamNode(Options{PrivKey: priv, Store: store, ControlPlanePubKey: enrollKey})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(restarted.trustedKeys) != 2 {
+ t.Fatalf("expected 2 trusted keys after restart, got %d", len(restarted.trustedKeys))
+ }
+ if !containsTrustedKey(restarted.trustedKeys, rotatedKey) {
+ t.Error("rotated key lost across restart")
+ }
+ if !containsTrustedKey(restarted.trustedKeys, enrollKey) {
+ t.Error("enrollment key missing after restart")
+ }
+}
+
// TestPruneTrustedKeys covers the trust-set floor: a node that runs past the
// grace period without witnessing a rotation must not age out its only key.
func TestPruneTrustedKeys(t *testing.T) {
diff --git a/internal/node/store.go b/internal/node/store.go
index ff88a8d6..c0d0dfda 100644
--- a/internal/node/store.go
+++ b/internal/node/store.go
@@ -39,6 +39,7 @@ const (
keyOidcIssuer = "oidc_issuer"
keyOidcClientID = "oidc_client_id"
keyOidcAudience = "oidc_audience"
+ keyTrustedKeys = "trusted_keys"
)
type Store struct {
@@ -239,6 +240,33 @@ func (s *Store) SaveControlPlaneURL(url string) error {
})
}
+// SaveTrustedKeys persists the full set of control plane public keys the
+// node currently trusts, so keys learned from rotation events or /keys
+// survive restarts (the singular mesh-config key only tracks enrollment).
+func (s *Store) SaveTrustedKeys(keys []TrustedKey) error {
+ data, err := json.Marshal(keys)
+ if err != nil {
+ return err
+ }
+ return s.db.Update(func(tx *bbolt.Tx) error {
+ b := tx.Bucket([]byte(bucketIdentity))
+ return b.Put([]byte(keyTrustedKeys), data)
+ })
+}
+
+func (s *Store) LoadTrustedKeys() ([]TrustedKey, error) {
+ var keys []TrustedKey
+ err := s.db.View(func(tx *bbolt.Tx) error {
+ b := tx.Bucket([]byte(bucketIdentity))
+ data := b.Get([]byte(keyTrustedKeys))
+ if len(data) == 0 {
+ return nil
+ }
+ return json.Unmarshal(data, &keys)
+ })
+ return keys, err
+}
+
func (s *Store) LoadControlPlaneURL() (string, error) {
var val []byte
err := s.db.View(func(tx *bbolt.Tx) error {
@@ -263,6 +291,7 @@ func (s *Store) ResetMeshIdentity() error {
keyOidcIssuer,
keyOidcClientID,
keyOidcAudience,
+ keyTrustedKeys,
"control_plane_public_key",
"router_addresses",
"control_plane_url",
diff --git a/internal/node/store_test.go b/internal/node/store_test.go
index 1989136e..d2a1f385 100644
--- a/internal/node/store_test.go
+++ b/internal/node/store_test.go
@@ -16,12 +16,14 @@ package node
import (
"bytes"
+ "crypto/ed25519"
"errors"
"fmt"
"os"
"path/filepath"
"reflect"
"testing"
+ "time"
"github.com/libp2p/go-libp2p/core/peer"
"go.etcd.io/bbolt"
@@ -45,6 +47,57 @@ func TestStore_NewStore_And_Close(t *testing.T) {
}
}
+func TestStore_TrustedKeys(t *testing.T) {
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatalf("Failed to create store: %v", err)
+ }
+ defer func() { _ = store.Close() }()
+
+ // Empty store: no keys, no error
+ keys, err := store.LoadTrustedKeys()
+ if err != nil {
+ t.Fatalf("LoadTrustedKeys on empty store: %v", err)
+ }
+ if len(keys) != 0 {
+ t.Fatalf("expected no keys, got %d", len(keys))
+ }
+
+ pub1, _, _ := ed25519.GenerateKey(nil)
+ pub2, _, _ := ed25519.GenerateKey(nil)
+ want := []TrustedKey{
+ {Key: pub1, ReceivedAt: time.Now().Add(-time.Hour).UTC().Truncate(time.Second)},
+ {Key: pub2, ReceivedAt: time.Now().UTC().Truncate(time.Second)},
+ }
+ if err := store.SaveTrustedKeys(want); err != nil {
+ t.Fatalf("SaveTrustedKeys: %v", err)
+ }
+ got, err := store.LoadTrustedKeys()
+ if err != nil {
+ t.Fatalf("LoadTrustedKeys: %v", err)
+ }
+ if len(got) != len(want) {
+ t.Fatalf("got %d keys, want %d", len(got), len(want))
+ }
+ for i := range got {
+ if !got[i].Key.Equal(want[i].Key) || !got[i].ReceivedAt.Equal(want[i].ReceivedAt) {
+ t.Errorf("key %d: got %+v, want %+v", i, got[i], want[i])
+ }
+ }
+
+ // Mesh reset must clear the persisted trust set
+ if err := store.ResetMeshIdentity(); err != nil {
+ t.Fatalf("ResetMeshIdentity: %v", err)
+ }
+ got, err = store.LoadTrustedKeys()
+ if err != nil {
+ t.Fatalf("LoadTrustedKeys after reset: %v", err)
+ }
+ if len(got) != 0 {
+ t.Fatalf("expected trust set cleared after mesh reset, got %d keys", len(got))
+ }
+}
+
func TestStore_Identity(t *testing.T) {
store, err := NewStore(t.TempDir())
if err != nil {