Skip to content

Agent Mesh

auth: derive the rotation refresh from the credential's issuance key set

Antonio Ojea3679ef6dd1d0
diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.go
index 8874a826..bf02c01f 100644
--- a/internal/node/controlplane_sync.go
+++ b/internal/node/controlplane_sync.go
@@ -16,6 +16,7 @@ package node
import (
"context"
+ "crypto/ed25519"
"errors"
"fmt"
"math/rand"
@@ -47,11 +48,10 @@ func (n *SamNode) SyncControlPlane(ctx context.Context) error {
return errors.New("control plane URL not found in store")
}
var errs []error
- if addedKeys, err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {
+ if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {
errs = append(errs, fmt.Errorf("keys: %w", err))
- } else if len(n.GetIdentity()) > 0 && (n.rotationRefreshPending.Swap(false) || addedKeys) {
+ } else if n.identityPredatesRotation() {
if err := n.RefreshEnrollment(ctx); err != nil {
- n.rotationRefreshPending.Store(true)
errs = append(errs, fmt.Errorf("refresh enrollment after key rotation: %w", err))
}
}
@@ -71,34 +71,83 @@ func (n *SamNode) SyncControlPlane(ctx context.Context) error {
// about. Verification against the keys already trusted keeps whoever answers
// the URL from becoming the trust root; an empty answer is a failure so the
// set is never wiped.
-func (n *SamNode) syncTrustedKeys(ctx context.Context, controlPlaneURL string) (bool, error) {
+func (n *SamNode) syncTrustedKeys(ctx context.Context, controlPlaneURL string) error {
n.keysMu.RLock()
existing := append([]TrustedKey(nil), n.trustedKeys...)
n.keysMu.RUnlock()
if len(existing) == 0 {
- return false, errors.New("no trusted control plane keys to verify /keys against")
+ return errors.New("no trusted control plane keys to verify /keys against")
}
keys, err := FetchControlPlaneKeys(ctx, controlPlaneURL, publicKeysOf(existing))
if err != nil {
- return false, err
+ return err
}
if len(keys) == 0 {
- return false, errors.New("/keys returned no keys")
+ return errors.New("/keys returned no keys")
}
merged := mergeTrustedKeys(existing, keys, time.Now())
- addedKeys := false
n.keysMu.Lock()
- for _, key := range keys {
- if !containsTrustedKey(n.trustedKeys, key) {
- addedKeys = true
- }
- }
n.trustedKeys = merged
snapshot := append([]TrustedKey(nil), merged...)
n.keysMu.Unlock()
n.persistTrustedKeys(snapshot)
logger.Debugf("Synced %d valid control plane keys", len(merged))
- return addedKeys, nil
+ return nil
+}
+
+// identityPredatesRotation reports whether a trusted key exists that was not
+// known when the stored identity was issued: the identity is then signed by
+// a retiring key and must be refreshed before that key leaves its grace
+// period. Both sides of the comparison are persisted, so the answer is the
+// same after a restart. An identity from a build that recorded no set is
+// checked against the stored mesh-config key, the one enrollment handed out
+// with it. Nothing enrolled means nothing to refresh.
+func (n *SamNode) identityPredatesRotation() bool {
+ if n.Store == nil || len(n.GetIdentity()) == 0 {
+ return false
+ }
+ issuance, err := n.Store.LoadIdentityKeySet()
+ if err != nil {
+ logger.Warnf("Could not load the identity's key set: %v", err)
+ return false
+ }
+ if len(issuance) == 0 {
+ enrollmentKey, _, err := n.Store.LoadMeshConfig()
+ if err != nil || len(enrollmentKey) != ed25519.PublicKeySize {
+ return false
+ }
+ issuance = []ed25519.PublicKey{ed25519.PublicKey(enrollmentKey)}
+ }
+ n.keysMu.RLock()
+ defer n.keysMu.RUnlock()
+ for _, tk := range n.trustedKeys {
+ if !containsPublicKey(issuance, tk.Key) {
+ return true
+ }
+ }
+ return false
+}
+
+func containsPublicKey(keys []ed25519.PublicKey, key ed25519.PublicKey) bool {
+ for _, candidate := range keys {
+ if candidate.Equal(key) {
+ return true
+ }
+ }
+ return false
+}
+
+// recordIdentityKeySet pins the trusted set to the identity just adopted.
+func (n *SamNode) recordIdentityKeySet() {
+ if n.Store == nil {
+ return
+ }
+ n.keysMu.RLock()
+ keys := publicKeysOf(n.trustedKeys)
+ n.keysMu.RUnlock()
+ if err := n.Store.SaveIdentityKeySet(keys); err != nil {
+ logger.Errorf("Failed to persist the identity's key set: %v", err)
+ }
}
// syncMeshInfo reads /info. The router addresses are persisted for the next
diff --git a/internal/node/controlplane_test.go b/internal/node/controlplane_test.go
index ed5fc40a..97b491c7 100644
--- a/internal/node/controlplane_test.go
+++ b/internal/node/controlplane_test.go
@@ -135,10 +135,8 @@ func TestSyncTrustedKeys(t *testing.T) {
srv := keysServer(t, []ed25519.PublicKey{retiredPub, currentPub}, []ed25519.PrivateKey{retiredPriv, currentPriv})
defer srv.Close()
n := newNode(t)
- if addedKeys, err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {
+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {
t.Fatalf("syncTrustedKeys: %v", err)
- } else if !addedKeys {
- t.Fatal("syncTrustedKeys must report the newly learned key")
}
if len(n.trustedKeys) != 2 || !containsTrustedKey(n.trustedKeys, retiredPub) || !containsTrustedKey(n.trustedKeys, currentPub) {
t.Errorf("trust set = %d keys, want retired and current", len(n.trustedKeys))
@@ -150,11 +148,8 @@ func TestSyncTrustedKeys(t *testing.T) {
if len(stored) != 2 {
t.Errorf("persisted %d keys, want 2", len(stored))
}
- if n.rotationRefreshPending.Load() {
- t.Error("adopting keys alone must not queue a credential refresh")
- }
- if addedKeys, err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil || addedKeys {
- t.Fatalf("unchanged key set: addedKeys = %v, err = %v", addedKeys, err)
+ if n.identityPredatesRotation() {
+ t.Error("a node without an identity has nothing to refresh")
}
})
@@ -162,7 +157,7 @@ func TestSyncTrustedKeys(t *testing.T) {
srv := keysServer(t, []ed25519.PublicKey{strangerPub}, []ed25519.PrivateKey{strangerPriv})
defer srv.Close()
n := newNode(t)
- if _, err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {
+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {
t.Fatal("a /keys answer signed only by an unknown key must not be adopted")
}
if len(n.trustedKeys) != 1 || !n.trustedKeys[0].Key.Equal(currentPub) {
@@ -174,7 +169,7 @@ func TestSyncTrustedKeys(t *testing.T) {
srv := keysServer(t, []ed25519.PublicKey{currentPub, strangerPub}, nil)
defer srv.Close()
n := newNode(t)
- if _, err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {
+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {
t.Fatal("an unsigned /keys answer must not be adopted")
}
if containsTrustedKey(n.trustedKeys, strangerPub) {
@@ -187,7 +182,7 @@ func TestSyncTrustedKeys(t *testing.T) {
defer srv.Close()
n := newNode(t)
n.trustedKeys = nil
- if _, err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {
+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {
t.Fatal("with no trusted key there is nothing to verify /keys against")
}
})
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index e496ae5f..6f93e3e5 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -81,9 +81,10 @@ func (n *SamNode) Enroll(ctx context.Context, controlPlaneURL string, jwt string
// control plane too old to serve /keys still enrolls with one key.
func (n *SamNode) adoptEnrolledKeys(ctx context.Context, controlPlaneURL string, newest []byte) {
n.addTrustedKey(ed25519.PublicKey(newest))
- if _, err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {
+ if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {
logger.Warnf("Could not fetch the control plane's full key set after enrollment (continuing with the enrollment key only): %v", err)
}
+ n.recordIdentityKeySet()
}
// enrollHTTP performs the HTTP half of enrollment for an explicit peer
diff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.go
index ac9524e4..4e8628a5 100644
--- a/internal/node/enroll_test.go
+++ b/internal/node/enroll_test.go
@@ -39,9 +39,13 @@ import (
"google.golang.org/protobuf/proto"
)
-func TestEnrollmentKeyAdoptionDoesNotQueueRefresh(t *testing.T) {
+// Enrollment hands out a biscuit signed by the current key and then learns
+// the full key set, grace key included. Neither step is a rotation the new
+// biscuit predates; a rotation that follows is.
+func TestEnrollmentPinsIdentityKeySet(t *testing.T) {
currentPub, currentPriv := mustGenerateKey(t)
gracePub, gracePriv := mustGenerateKey(t)
+ rotatedPub, _ := mustGenerateKey(t)
store, err := NewStore(t.TempDir())
if err != nil {
t.Fatal(err)
@@ -78,16 +82,29 @@ func TestEnrollmentKeyAdoptionDoesNotQueueRefresh(t *testing.T) {
if !bytes.Equal(node.GetIdentity(), credential) || len(node.trustedKeys) != 2 {
t.Fatal("enrollment must adopt the credential and both trusted keys")
}
- if node.rotationRefreshPending.Load() {
- t.Error("enrollment key adoption queued a redundant refresh")
+ if node.identityPredatesRotation() {
+ t.Error("a freshly enrolled identity predates no rotation")
}
if err := node.SyncControlPlane(context.Background()); err != nil {
t.Fatalf("sync after enrollment: %v", err)
}
- node.rotationRefreshPending.Store(true)
- node.adoptEnrolledKeys(context.Background(), server.URL, currentPub)
- if !node.rotationRefreshPending.Load() {
- t.Fatal("enrollment key adoption cleared a pending rotation refresh")
+ pinned, err := store.LoadIdentityKeySet()
+ if err != nil || len(pinned) != 2 {
+ t.Fatalf("identity key set = %d keys, err %v; want the grace and current keys", len(pinned), err)
+ }
+
+ node.handleKeyRotationEvent(&api.MeshEvent{NewPublicKey: rotatedPub})
+ if !node.identityPredatesRotation() {
+ t.Fatal("a key learned after issuance must mark the identity for refresh")
+ }
+ restarted := &SamNode{Store: store}
+ if stored, err := store.LoadTrustedKeys(); err != nil {
+ t.Fatal(err)
+ } else {
+ restarted.trustedKeys = stored
+ }
+ if !restarted.identityPredatesRotation() {
+ t.Fatal("the refresh decision must survive a restart")
}
}
diff --git a/internal/node/node.go b/internal/node/node.go
index 4714308c..972011eb 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -186,10 +186,13 @@ type SamNode struct {
currentRelays []peer.AddrInfo
reprovideTrigger chan struct{}
controlPlaneSyncTrigger chan struct{}
- rotationRefreshPending atomic.Bool
- BiscuitTimeout time.Duration
- cachedIdentity atomic.Value
- logger *golog.ZapEventLogger
+ // refreshMu serializes credential issuance: the control plane redeems
+ // only the last biscuit it issued, so two refreshes in flight would
+ // invalidate each other.
+ refreshMu sync.Mutex
+ BiscuitTimeout time.Duration
+ cachedIdentity atomic.Value
+ logger *golog.ZapEventLogger
// metricsRegistry holds this node's state collector; see metricsHandler.
metricsOnce sync.Once
@@ -1111,6 +1114,9 @@ func (e *RefreshError) Error() string {
// RefreshEnrollment trades the expiring biscuit token for a new one using a cryptographic challenge.
func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
+ n.refreshMu.Lock()
+ defer n.refreshMu.Unlock()
+
// 1. Fetch current biscuit
currentBiscuit, err := n.Store.LoadIdentity()
if err != nil {
@@ -1226,6 +1232,7 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {
if err := n.Store.SaveIdentityExpiration(refreshResp.ExpiresAt); err != nil {
return fmt.Errorf("failed to save refreshed expiration: %w", err)
}
+ n.recordIdentityKeySet()
return nil
}
@@ -1398,17 +1405,16 @@ func containsTrustedKey(keys []TrustedKey, key ed25519.PublicKey) bool {
// addTrustedKey appends a control plane public key to the trust set (no-op
// on duplicates) and persists the updated set so it survives restarts.
-func (n *SamNode) addTrustedKey(key ed25519.PublicKey) bool {
+func (n *SamNode) addTrustedKey(key ed25519.PublicKey) {
n.keysMu.Lock()
if containsTrustedKey(n.trustedKeys, key) {
n.keysMu.Unlock()
- return false
+ return
}
n.trustedKeys = append(n.trustedKeys, TrustedKey{Key: key, ReceivedAt: time.Now()})
snapshot := append([]TrustedKey(nil), n.trustedKeys...)
n.keysMu.Unlock()
n.persistTrustedKeys(snapshot)
- return true
}
func (n *SamNode) persistTrustedKeys(keys []TrustedKey) {
@@ -1426,9 +1432,7 @@ func (n *SamNode) handleKeyRotationEvent(event *api.MeshEvent) {
return
}
logger.Infow("[Mesh Event] key rotation received", "event", meshEventKeyRotation, "key", fmt.Sprintf("%x", event.NewPublicKey))
- if n.addTrustedKey(ed25519.PublicKey(event.NewPublicKey)) && len(n.GetIdentity()) > 0 {
- n.rotationRefreshPending.Store(true)
- }
+ n.addTrustedKey(ed25519.PublicKey(event.NewPublicKey))
n.triggerControlPlaneSync()
}
diff --git a/internal/node/node_test.go b/internal/node/node_test.go
index 1a9af9e7..a22bcaef 100644
--- a/internal/node/node_test.go
+++ b/internal/node/node_test.go
@@ -202,8 +202,8 @@ func TestHandleKeyRotationEvent(t *testing.T) {
if len(node.trustedKeys) != 1 {
t.Errorf("Expected 1 trusted key, got %d", len(node.trustedKeys))
}
- if node.rotationRefreshPending.Load() {
- t.Error("an unenrolled node must not queue a credential refresh")
+ if node.identityPredatesRotation() {
+ t.Error("an unenrolled node has nothing to refresh")
}
}
diff --git a/internal/node/refresh_test.go b/internal/node/refresh_test.go
index 6e2ab5c8..38d61e5f 100644
--- a/internal/node/refresh_test.go
+++ b/internal/node/refresh_test.go
@@ -19,9 +19,12 @@ import (
"context"
"crypto/ed25519"
"crypto/rand"
+ "encoding/base64"
"errors"
"net/http"
"net/http/httptest"
+ "strings"
+ "sync"
"sync/atomic"
"testing"
"time"
@@ -86,6 +89,9 @@ func newRefreshHarness(t *testing.T, refresh http.HandlerFunc) *refreshHarness {
if err := store.SaveIdentity(current); err != nil {
t.Fatal(err)
}
+ if err := store.SaveIdentityKeySet([]ed25519.PublicKey{cpPub}); err != nil {
+ t.Fatal(err)
+ }
mux := http.NewServeMux()
mux.HandleFunc("/refresh", refresh)
@@ -173,6 +179,121 @@ func TestRotationRefreshDuringControlPlaneSync(t *testing.T) {
}
}
+// A rotation learned before a restart must still be acted on after it: the
+// persisted key set is then unchanged, so the decision cannot hinge on
+// noticing a new key. Identities issued by builds that recorded no key set
+// fall back to the enrollment key stored in the mesh config.
+func TestRotationRefreshSurvivesRestart(t *testing.T) {
+ for _, legacy := range []bool{false, true} {
+ name := "recorded key set"
+ if legacy {
+ name = "pre-upgrade identity"
+ }
+ t.Run(name, func(t *testing.T) {
+ newPub, newPriv := mustGenerateKey(t)
+ var fresh []byte
+ var refreshes atomic.Int32
+ refresh := func(w http.ResponseWriter, request *http.Request) {
+ if refreshes.Add(1) == 1 {
+ http.Error(w, "temporarily unavailable", http.StatusServiceUnavailable)
+ return
+ }
+ writeRefreshResponse(t, w, fresh)
+ }
+ harness := newRefreshHarness(t, refresh)
+ fresh = mintRoleBiscuit(t, newPriv, harness.peerID, api.RoleNode)
+ store := harness.node.Store
+ if legacy {
+ if err := store.SaveIdentityKeySet(nil); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveMeshConfig(harness.cpPub, nil); err != nil {
+ t.Fatal(err)
+ }
+ }
+ mux := http.NewServeMux()
+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{harness.cpPub, newPub}, []ed25519.PrivateKey{harness.cpPriv, newPriv}))
+ mux.HandleFunc("/refresh", refresh)
+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{}))
+ mux.HandleFunc("/policies", protoHandler(t, &api.PolicyConfigGetResponse{}))
+ server := httptest.NewServer(mux)
+ defer server.Close()
+ if err := store.SaveControlPlaneURL(server.URL); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveTrustedKeys([]TrustedKey{{Key: harness.cpPub, ReceivedAt: time.Now()}}); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := harness.node.SyncControlPlane(context.Background()); err == nil {
+ t.Fatal("sync must report the failed credential refresh")
+ }
+ harness.storedIdentityUnchanged(t)
+
+ // Restart: only what the store holds carries over.
+ restarted, err := NewSamNode(Options{PrivKey: GetOrGenerateKey(store), Store: store, ControlPlanePubKey: harness.cpPub, RequiredRole: api.RoleNode, BiscuitTimeout: time.Second})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(restarted.trustedKeys) != 2 {
+ t.Fatalf("restarted node loaded %d trusted keys, want the persisted pair", len(restarted.trustedKeys))
+ }
+ if err := restarted.SyncControlPlane(context.Background()); err != nil {
+ t.Fatalf("refresh after restart: %v", err)
+ }
+ if !bytes.Equal(restarted.GetIdentity(), fresh) {
+ t.Fatal("restarted node kept a credential signed by the retiring key")
+ }
+ if err := restarted.SyncControlPlane(context.Background()); err != nil {
+ t.Fatalf("settled sync: %v", err)
+ }
+ if got := refreshes.Load(); got != 2 {
+ t.Fatalf("refresh attempts = %d, want one failure before and one success after the restart", got)
+ }
+ })
+ }
+}
+
+// The control plane redeems only the last biscuit it issued, so the rotation
+// sync and the expiry renewal loop must not refresh at the same time: the
+// loser would present an already-rotated biscuit and be refused as a replay.
+func TestConcurrentRefreshesAreSerialized(t *testing.T) {
+ var mu sync.Mutex
+ var lastIssued []byte
+ var harness *refreshHarness
+ harness = newRefreshHarness(t, func(w http.ResponseWriter, request *http.Request) {
+ presented, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(request.Header.Get("Authorization"), "Bearer "))
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if !bytes.Equal(presented, lastIssued) {
+ http.Error(w, "Biscuit already rotated", http.StatusUnauthorized)
+ return
+ }
+ lastIssued = mintRoleBiscuit(t, harness.cpPriv, harness.peerID, api.RoleNode)
+ writeRefreshResponse(t, w, lastIssued)
+ })
+ lastIssued = harness.identity
+
+ errs := make(chan error, 2)
+ for range 2 {
+ go func() { errs <- harness.node.RefreshEnrollment(context.Background()) }()
+ }
+ for range 2 {
+ if err := <-errs; err != nil {
+ t.Fatalf("concurrent refresh: %v", err)
+ }
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if !bytes.Equal(harness.node.GetIdentity(), lastIssued) {
+ t.Fatal("node does not hold the last biscuit the control plane issued")
+ }
+}
+
func (h *refreshHarness) storedIdentityUnchanged(t *testing.T) {
t.Helper()
stored, err := h.node.Store.LoadIdentity()
diff --git a/internal/node/store.go b/internal/node/store.go
index 4d039d9a..12a9b158 100644
--- a/internal/node/store.go
+++ b/internal/node/store.go
@@ -15,6 +15,7 @@
package node
import (
+ "crypto/ed25519"
"encoding/json"
"errors"
"fmt"
@@ -39,6 +40,7 @@ const (
keyOidcClientID = "oidc_client_id"
keyOidcAudience = "oidc_audience"
keyTrustedKeys = "trusted_keys"
+ keyIdentityKeys = "identity_key_set"
)
type Store struct {
@@ -263,6 +265,35 @@ func (s *Store) LoadTrustedKeys() ([]TrustedKey, error) {
return keys, err
}
+// SaveIdentityKeySet records the control plane public keys known when the
+// stored identity was issued. A key that later appears outside this set is
+// a rotation the identity predates, which is why it survives restarts.
+func (s *Store) SaveIdentityKeySet(keys []ed25519.PublicKey) error {
+ data, err := json.Marshal(keys)
+ if err != nil {
+ return err
+ }
+ return s.db.Update(func(tx *bbolt.Tx) error {
+ b := tx.Bucket([]byte(bucketIdentity))
+ return b.Put([]byte(keyIdentityKeys), data)
+ })
+}
+
+// LoadIdentityKeySet returns the set saved by SaveIdentityKeySet, or nil when
+// the identity was issued by a build that did not record it.
+func (s *Store) LoadIdentityKeySet() ([]ed25519.PublicKey, error) {
+ var keys []ed25519.PublicKey
+ err := s.db.View(func(tx *bbolt.Tx) error {
+ b := tx.Bucket([]byte(bucketIdentity))
+ data := b.Get([]byte(keyIdentityKeys))
+ if len(data) == 0 {
+ return nil
+ }
+ return json.Unmarshal(data, &keys)
+ })
+ return keys, err
+}
+
func (s *Store) LoadControlPlaneURL() (string, error) {
var val []byte
err := s.db.View(func(tx *bbolt.Tx) error {
@@ -288,6 +319,7 @@ func (s *Store) ResetMeshIdentity() error {
keyOidcClientID,
keyOidcAudience,
keyTrustedKeys,
+ keyIdentityKeys,
"control_plane_public_key",
"router_addresses",
"control_plane_url",
diff --git a/internal/router/metrics.go b/internal/router/metrics.go
index ee50a481..b5ca189f 100644
--- a/internal/router/metrics.go
+++ b/internal/router/metrics.go
@@ -140,7 +140,7 @@ func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.dhtDesc, prometheus.GaugeValue, float64(r.DHT.RoutingTable().Size()))
r.keysMu.RLock()
- expiry := r.biscuitExpiration
+ expiry := r.credential.expiration
r.keysMu.RUnlock()
if !expiry.IsZero() {
ch <- prometheus.MustNewConstMetric(c.biscuitExpiryDesc, prometheus.GaugeValue, float64(expiry.Unix()))
diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.go
index 3e17ecfd..3b0f93a9 100644
--- a/internal/router/metrics_test.go
+++ b/internal/router/metrics_test.go
@@ -88,7 +88,7 @@ func TestRouterMetricsListener(t *testing.T) {
r.bannedPeers.Store(peer.ID("peer-x"), true)
expiry := time.Now().Add(time.Hour).Truncate(time.Second)
r.keysMu.Lock()
- r.biscuitExpiration = expiry
+ r.credential.expiration = expiry
r.keysMu.Unlock()
r.isReady.Store(true)
diff --git a/internal/router/router.go b/internal/router/router.go
index 702d9e43..87d35778 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -110,6 +110,54 @@ func (a *relayACL) AllowConnect(src peer.ID, srcAddr multiaddr.Multiaddr, dest p
return ok
}
+// credential is the router's own mesh identity as the control plane issued
+// it. issuedUnder is the trusted key set at issuance: a trusted key outside
+// it is a rotation the biscuit predates, so the biscuit must be refreshed
+// before that key's predecessor leaves its grace period. It is nil until the
+// first /keys sync after enrollment pins it, since enrollment hands out one
+// key and the sync learns the rest.
+type credential struct {
+ biscuit []byte
+ expiration time.Time
+ issuedUnder []ed25519.PublicKey
+}
+
+func newCredential(biscuit []byte, expiresAt int64, issuedUnder []ed25519.PublicKey) credential {
+ return credential{
+ biscuit: biscuit,
+ expiration: time.Unix(expiresAt, 0),
+ issuedUnder: append([]ed25519.PublicKey(nil), issuedUnder...),
+ }
+}
+
+// pin records trusted as the issuance set of a biscuit that has none yet.
+func (c *credential) pin(trusted []ed25519.PublicKey) {
+ if len(c.biscuit) > 0 && c.issuedUnder == nil {
+ c.issuedUnder = append([]ed25519.PublicKey(nil), trusted...)
+ }
+}
+
+// predatesRotation reports whether trusted holds a key unknown at issuance.
+func (c credential) predatesRotation(trusted []ed25519.PublicKey) bool {
+ return len(c.biscuit) > 0 && c.issuedUnder != nil && !subsetOf(trusted, c.issuedUnder)
+}
+
+func subsetOf(keys, set []ed25519.PublicKey) bool {
+ for _, key := range keys {
+ known := false
+ for _, candidate := range set {
+ if key.Equal(candidate) {
+ known = true
+ break
+ }
+ }
+ if !known {
+ return false
+ }
+ }
+ return true
+}
+
// Router represents a libp2p bootstrap/relay node.
type Router struct {
config Options
@@ -123,16 +171,19 @@ type Router struct {
// can open those streams.
handshakeLimiter *ratelimit.PeerRateLimiter
- // Keys & Identity
- biscuitToken []byte
- biscuitExpiration time.Time
+ // trustedPublicKeys is what the control plane signs with now, the set
+ // every peer credential is verified against; credential is this
+ // router's own. Both are guarded by keysMu.
trustedPublicKeys []ed25519.PublicKey
+ credential credential
keysMu sync.RWMutex
enrollMu sync.Mutex
privKey crypto.PrivKey
+ // refreshMu serializes credential issuance: the control plane redeems
+ // only the last biscuit it issued.
+ refreshMu sync.Mutex
- keysSyncTrigger chan struct{}
- rotationRefreshPending bool
+ keysSyncTrigger chan struct{}
// Control contexts
ctx context.Context
@@ -459,8 +510,7 @@ func (r *Router) enroll(peerID peer.ID) error {
}
r.keysMu.Lock()
- r.biscuitToken = enrollResp.BiscuitToken
- r.biscuitExpiration = time.Unix(enrollResp.Expiration, 0)
+ r.credential = newCredential(enrollResp.BiscuitToken, enrollResp.Expiration, nil)
r.trustedPublicKeys = []ed25519.PublicKey{enrollResp.ControlPlanePublicKey}
r.keysMu.Unlock()
@@ -604,8 +654,7 @@ func (r *Router) enrollBootstrap(peerID peer.ID) error {
}
r.keysMu.Lock()
- r.biscuitToken = enrollResp.BiscuitToken
- r.biscuitExpiration = time.Unix(enrollResp.Expiration, 0)
+ r.credential = newCredential(enrollResp.BiscuitToken, enrollResp.Expiration, nil)
r.trustedPublicKeys = []ed25519.PublicKey{enrollResp.ControlPlanePublicKey}
r.keysMu.Unlock()
@@ -692,26 +741,15 @@ func (r *Router) syncKeys() error {
}
r.keysMu.Lock()
- for _, newKey := range newKeys {
- known := false
- for _, oldKey := range r.trustedPublicKeys {
- if newKey.Equal(oldKey) {
- known = true
- break
- }
- }
- if !known && len(r.biscuitToken) > 0 {
- r.rotationRefreshPending = true
- }
- }
r.trustedPublicKeys = newKeys
+ r.credential.pin(newKeys)
+ refresh := r.credential.predatesRotation(newKeys)
r.keysMu.Unlock()
- if r.rotationRefreshPending {
+ if refresh {
if err := r.RefreshEnrollment(r.ctx); err != nil {
return fmt.Errorf("refresh enrollment after key rotation: %w", err)
}
- r.rotationRefreshPending = false
}
logger.Debugf("Synced %d valid public keys from control plane", len(newKeys))
@@ -872,7 +910,7 @@ func (r *Router) runLeaseRenewalLoop() {
func (r *Router) renewLease() {
for attempt := 0; attempt < 2; attempt++ {
r.keysMu.RLock()
- biscuit := r.biscuitToken
+ biscuit := r.credential.biscuit
r.keysMu.RUnlock()
if len(biscuit) == 0 {
@@ -1190,7 +1228,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
// Send mutual response (our biscuit)
r.keysMu.RLock()
- ourBiscuit := r.biscuitToken
+ ourBiscuit := r.credential.biscuit
r.keysMu.RUnlock()
writer := msgio.NewVarintWriter(s)
@@ -1213,8 +1251,11 @@ func (r *Router) performMutualAuth(s network.Stream) error {
_ = s.SetDeadline(time.Now().Add(5 * time.Second))
// Send our biscuit
+ r.keysMu.RLock()
+ ourBiscuit := r.credential.biscuit
+ r.keysMu.RUnlock()
writer := msgio.NewVarintWriter(s)
- authFrame := &api.AuthFrame{Biscuit: r.biscuitToken}
+ authFrame := &api.AuthFrame{Biscuit: ourBiscuit}
data, _ := proto.Marshal(authFrame)
if err := writer.WriteMsg(data); err != nil {
return fmt.Errorf("write mutual auth frame: %w", err)
@@ -1328,8 +1369,11 @@ func isLoopbackOrLinkLocal(addr multiaddr.Multiaddr) bool {
// RefreshEnrollment trades the expiring biscuit token for a new one using a cryptographic challenge.
func (r *Router) RefreshEnrollment(ctx context.Context) error {
+ r.refreshMu.Lock()
+ defer r.refreshMu.Unlock()
+
r.keysMu.RLock()
- currentBiscuit := r.biscuitToken
+ currentBiscuit := r.credential.biscuit
r.keysMu.RUnlock()
if len(currentBiscuit) == 0 {
@@ -1418,8 +1462,7 @@ func (r *Router) RefreshEnrollment(ctx context.Context) error {
// Update local biscuit token and expiration under lock
r.keysMu.Lock()
- r.biscuitToken = refreshResp.BiscuitToken
- r.biscuitExpiration = time.Unix(refreshResp.ExpiresAt, 0)
+ r.credential = newCredential(refreshResp.BiscuitToken, refreshResp.ExpiresAt, r.trustedPublicKeys)
r.keysMu.Unlock()
logger.Infof("Router biscuit token refreshed successfully.")
@@ -1447,7 +1490,7 @@ func (r *Router) runBiscuitRenewalLoop() {
select {
case <-ticker.C:
r.keysMu.RLock()
- expiration := r.biscuitExpiration
+ expiration := r.credential.expiration
r.keysMu.RUnlock()
if !expiration.IsZero() && time.Until(expiration) < api.BiscuitTokenTTL/5 { // 80% elapsed lifespan of 24h (remaining time < 20% of TTL)
diff --git a/internal/router/router_test.go b/internal/router/router_test.go
index eec83e02..e817c825 100644
--- a/internal/router/router_test.go
+++ b/internal/router/router_test.go
@@ -452,8 +452,8 @@ func TestRouterProactiveTokenRefresh(t *testing.T) {
defer func() { _ = r.Close() }()
// Capture initial token and expiration
- initialToken := r.biscuitToken
- initialExpiration := r.biscuitExpiration
+ initialToken := r.credential.biscuit
+ initialExpiration := r.credential.expiration
if len(initialToken) == 0 {
t.Fatal("initial router biscuit token is empty")
@@ -466,11 +466,11 @@ func TestRouterProactiveTokenRefresh(t *testing.T) {
}
// Assert that token and expiration updated
- if bytes.Equal(r.biscuitToken, initialToken) {
+ if bytes.Equal(r.credential.biscuit, initialToken) {
t.Error("biscuit token did not change after refresh")
}
- if !r.biscuitExpiration.After(initialExpiration) {
- t.Errorf("expected refreshed expiration %v to be after initial expiration %v", r.biscuitExpiration, initialExpiration)
+ if !r.credential.expiration.After(initialExpiration) {
+ t.Errorf("expected refreshed expiration %v to be after initial expiration %v", r.credential.expiration, initialExpiration)
}
}
@@ -514,19 +514,19 @@ func TestRouterLeaseRenewalReEnrollOn401(t *testing.T) {
// Corrupt router's biscuit token to simulate key rotation/expiration
r.keysMu.Lock()
- r.biscuitToken = []byte("invalid-corrupted-biscuit")
+ r.credential.biscuit = []byte("invalid-corrupted-biscuit")
r.keysMu.Unlock()
// Call renewLease which should fail with 401, trigger reEnroll, and succeed
r.renewLease()
- // Verify that biscuitToken was replaced with a valid non-corrupted token
+ // Verify that the biscuit was replaced with a valid non-corrupted token
r.keysMu.RLock()
- currentToken := r.biscuitToken
+ currentToken := r.credential.biscuit
r.keysMu.RUnlock()
if bytes.Equal(currentToken, []byte("invalid-corrupted-biscuit")) {
- t.Error("expected biscuitToken to be updated after 401 re-enrollment, but it remained corrupted")
+ t.Error("expected the biscuit to be updated after 401 re-enrollment, but it remained corrupted")
}
}
@@ -545,9 +545,9 @@ func TestRouterLeaseRenewalRepeated401Terminates(t *testing.T) {
defer func() { _ = h.Close() }()
r := &Router{
- Host: h,
- privKey: h.Peerstore().PrivKey(h.ID()),
- biscuitToken: []byte("dummy-biscuit"),
+ Host: h,
+ privKey: h.Peerstore().PrivKey(h.ID()),
+ credential: credential{biscuit: []byte("dummy-biscuit")},
config: Options{
ControlPlaneURL: ts.URL,
},
@@ -612,7 +612,7 @@ func TestRouterProactiveRefreshReEnrollOn401(t *testing.T) {
// Corrupt router's biscuit token to simulate key rotation
r.keysMu.Lock()
- r.biscuitToken = []byte("stale-biscuit-signed-by-purged-key")
+ r.credential.biscuit = []byte("stale-biscuit-signed-by-purged-key")
r.keysMu.Unlock()
// Trigger proactive refresh which should fail with 401, trigger reEnroll, and succeed
@@ -621,13 +621,13 @@ func TestRouterProactiveRefreshReEnrollOn401(t *testing.T) {
t.Fatalf("expected RefreshEnrollment to recover via reEnroll, but got error: %v", err)
}
- // Verify that biscuitToken was replaced with a valid fresh token
+ // Verify that the biscuit was replaced with a valid fresh token
r.keysMu.RLock()
- currentToken := r.biscuitToken
+ currentToken := r.credential.biscuit
r.keysMu.RUnlock()
if bytes.Equal(currentToken, []byte("stale-biscuit-signed-by-purged-key")) {
- t.Error("expected biscuitToken to be updated after 401 refresh fallback, but it remained stale")
+ t.Error("expected the biscuit to be updated after 401 refresh fallback, but it remained stale")
}
}
@@ -872,7 +872,7 @@ func TestPerformMutualAuth(t *testing.T) {
r := &Router{
Host: clientHost,
- biscuitToken: []byte("client-biscuit"),
+ credential: credential{biscuit: []byte("client-biscuit")},
trustedPublicKeys: []ed25519.PublicKey{oldPub, newPub},
config: Options{
BiscuitTimeout: time.Second,
@@ -936,7 +936,7 @@ func TestHandleAuthHandshakeBoundsUnauthenticatedPeers(t *testing.T) {
t.Fatal(err)
}
r.Host = serverHost
- r.biscuitToken = []byte("router-biscuit")
+ r.credential.biscuit = []byte("router-biscuit")
r.trustedPublicKeys = []ed25519.PublicKey{cpPub}
serverHost.SetStreamHandler(api.AuthProtocolID, r.HandleAuthHandshake)
if err := clientHost.Connect(ctx, peer.AddrInfo{ID: serverHost.ID(), Addrs: serverHost.Addrs()}); err != nil {
diff --git a/internal/router/trust_test.go b/internal/router/trust_test.go
index 3f12bef2..d4eab4fd 100644
--- a/internal/router/trust_test.go
+++ b/internal/router/trust_test.go
@@ -19,10 +19,12 @@ import (
"context"
"crypto/ed25519"
"crypto/rand"
+ "encoding/base64"
"errors"
"net/http"
"net/http/httptest"
"strings"
+ "sync"
"sync/atomic"
"testing"
"time"
@@ -274,7 +276,7 @@ func TestSyncKeysRefreshesOnRotation(t *testing.T) {
router := &Router{
ctx: context.Background(),
privKey: priv,
- biscuitToken: mintRouterBiscuit(t, oldPriv, peerID, api.RoleRouter),
+ credential: credential{biscuit: mintRouterBiscuit(t, oldPriv, peerID, api.RoleRouter), issuedUnder: []ed25519.PublicKey{oldPub}},
trustedPublicKeys: []ed25519.PublicKey{oldPub},
config: Options{ControlPlaneURL: server.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},
}
@@ -284,7 +286,7 @@ func TestSyncKeysRefreshesOnRotation(t *testing.T) {
if err := router.syncKeys(); err != nil {
t.Fatalf("retry refresh: %v", err)
}
- if !bytes.Equal(router.biscuitToken, fresh) {
+ if !bytes.Equal(router.credential.biscuit, fresh) {
t.Fatal("router kept a credential signed by the retiring key")
}
if err := router.syncKeys(); err != nil {
@@ -293,6 +295,85 @@ func TestSyncKeysRefreshesOnRotation(t *testing.T) {
if got := refreshes.Load(); got != 2 {
t.Fatalf("refresh attempts = %d, want one failure and one success", got)
}
+
+ // A freshly enrolled router knows one key; its first sync learns the
+ // grace key too. That is not a rotation its new biscuit predates.
+ enrolled := &Router{
+ ctx: context.Background(),
+ privKey: priv,
+ credential: credential{biscuit: fresh},
+ trustedPublicKeys: []ed25519.PublicKey{newPub},
+ config: Options{ControlPlaneURL: server.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},
+ }
+ if err := enrolled.syncKeys(); err != nil {
+ t.Fatalf("first sync after enrollment: %v", err)
+ }
+ if got := refreshes.Load(); got != 2 {
+ t.Fatalf("first sync after enrollment refreshed (%d attempts total)", got)
+ }
+ if len(enrolled.credential.issuedUnder) != 2 {
+ t.Fatalf("issuance set = %d keys, want the synced pair", len(enrolled.credential.issuedUnder))
+ }
+}
+
+// The control plane redeems only the last biscuit it issued, so the keys
+// loop, the renewal loop and lease recovery must not refresh concurrently.
+func TestRouterConcurrentRefreshesAreSerialized(t *testing.T) {
+ cpPub, cpPriv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ priv, _, err := crypto.GenerateEd25519Key(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ peerID, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var mu sync.Mutex
+ lastIssued := mintRouterBiscuit(t, cpPriv, peerID, api.RoleRouter)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+ presented, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(request.Header.Get("Authorization"), "Bearer "))
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if !bytes.Equal(presented, lastIssued) {
+ http.Error(w, "Biscuit already rotated", http.StatusUnauthorized)
+ return
+ }
+ lastIssued = mintRouterBiscuit(t, cpPriv, peerID, api.RoleRouter)
+ body, err := proto.Marshal(&api.TokenRefreshResponse{BiscuitToken: lastIssued, ExpiresAt: time.Now().Add(time.Hour).Unix()})
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ _, _ = w.Write(body)
+ }))
+ defer server.Close()
+ router := &Router{
+ privKey: priv,
+ credential: credential{biscuit: lastIssued},
+ trustedPublicKeys: []ed25519.PublicKey{cpPub},
+ config: Options{ControlPlaneURL: server.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},
+ }
+ errs := make(chan error, 2)
+ for range 2 {
+ go func() { errs <- router.RefreshEnrollment(context.Background()) }()
+ }
+ for range 2 {
+ if err := <-errs; err != nil {
+ t.Fatalf("concurrent refresh: %v", err)
+ }
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if !bytes.Equal(router.credential.biscuit, lastIssued) {
+ t.Fatal("router does not hold the last biscuit the control plane issued")
+ }
}
func mintRouterBiscuit(t *testing.T, priv ed25519.PrivateKey, peerID peer.ID, role string) []byte {
@@ -353,7 +434,7 @@ func TestRouterRefreshEnrollmentHardening(t *testing.T) {
newRouter := func() *Router {
return &Router{
privKey: priv,
- biscuitToken: current,
+ credential: credential{biscuit: current},
trustedPublicKeys: []ed25519.PublicKey{cpPub},
config: Options{ControlPlaneURL: srv.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},
}
@@ -375,7 +456,7 @@ func TestRouterRefreshEnrollmentHardening(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "403") {
t.Fatalf("RefreshEnrollment = %v, want a 403 error", err)
}
- if !bytes.Equal(r.biscuitToken, current) {
+ if !bytes.Equal(r.credential.biscuit, current) {
t.Error("the current biscuit must be kept on a refused refresh")
}
})
@@ -391,7 +472,7 @@ func TestRouterRefreshEnrollmentHardening(t *testing.T) {
if err := r.RefreshEnrollment(context.Background()); err == nil {
t.Fatal("RefreshEnrollment adopted a token it must have refused")
}
- if !bytes.Equal(r.biscuitToken, current) {
+ if !bytes.Equal(r.credential.biscuit, current) {
t.Error("the current biscuit must be kept when the refreshed one is refused")
}
})
@@ -404,7 +485,7 @@ func TestRouterRefreshEnrollmentHardening(t *testing.T) {
if err := r.RefreshEnrollment(context.Background()); err != nil {
t.Fatalf("RefreshEnrollment: %v", err)
}
- if !bytes.Equal(r.biscuitToken, fresh) {
+ if !bytes.Equal(r.credential.biscuit, fresh) {
t.Error("a valid refreshed token must replace the current one")
}
})