Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
auth: derive the rotation refresh from the credential's issuance key set
Antonio Ojea3679ef6dd1d0
diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.goindex 8874a826..bf02c01f 100644--- a/internal/node/controlplane_sync.go+++ b/internal/node/controlplane_sync.go@@ -16,6 +16,7 @@ package nodeimport ("context"+ "crypto/ed25519""errors""fmt""math/rand"@@ -47,11 +48,10 @@ func (n *SamNode) SyncControlPlane(ctx context.Context) error {return errors.New("control plane URL not found in store")}var errs []error- if addedKeys, err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {+ if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {errs = append(errs, fmt.Errorf("keys: %w", err))- } else if len(n.GetIdentity()) > 0 && (n.rotationRefreshPending.Swap(false) || addedKeys) {+ } else if n.identityPredatesRotation() {if err := n.RefreshEnrollment(ctx); err != nil {- n.rotationRefreshPending.Store(true)errs = append(errs, fmt.Errorf("refresh enrollment after key rotation: %w", err))}}@@ -71,34 +71,83 @@ func (n *SamNode) SyncControlPlane(ctx context.Context) error {// about. Verification against the keys already trusted keeps whoever answers// the URL from becoming the trust root; an empty answer is a failure so the// set is never wiped.-func (n *SamNode) syncTrustedKeys(ctx context.Context, controlPlaneURL string) (bool, error) {+func (n *SamNode) syncTrustedKeys(ctx context.Context, controlPlaneURL string) error {n.keysMu.RLock()existing := append([]TrustedKey(nil), n.trustedKeys...)n.keysMu.RUnlock()if len(existing) == 0 {- return false, errors.New("no trusted control plane keys to verify /keys against")+ return errors.New("no trusted control plane keys to verify /keys against")}keys, err := FetchControlPlaneKeys(ctx, controlPlaneURL, publicKeysOf(existing))if err != nil {- return false, err+ return err}if len(keys) == 0 {- return false, errors.New("/keys returned no keys")+ return errors.New("/keys returned no keys")}merged := mergeTrustedKeys(existing, keys, time.Now())- addedKeys := falsen.keysMu.Lock()- for _, key := range keys {- if !containsTrustedKey(n.trustedKeys, key) {- addedKeys = true- }- }n.trustedKeys = mergedsnapshot := append([]TrustedKey(nil), merged...)n.keysMu.Unlock()n.persistTrustedKeys(snapshot)logger.Debugf("Synced %d valid control plane keys", len(merged))- return addedKeys, nil+ return nil+}++// identityPredatesRotation reports whether a trusted key exists that was not+// known when the stored identity was issued: the identity is then signed by+// a retiring key and must be refreshed before that key leaves its grace+// period. Both sides of the comparison are persisted, so the answer is the+// same after a restart. An identity from a build that recorded no set is+// checked against the stored mesh-config key, the one enrollment handed out+// with it. Nothing enrolled means nothing to refresh.+func (n *SamNode) identityPredatesRotation() bool {+ if n.Store == nil || len(n.GetIdentity()) == 0 {+ return false+ }+ issuance, err := n.Store.LoadIdentityKeySet()+ if err != nil {+ logger.Warnf("Could not load the identity's key set: %v", err)+ return false+ }+ if len(issuance) == 0 {+ enrollmentKey, _, err := n.Store.LoadMeshConfig()+ if err != nil || len(enrollmentKey) != ed25519.PublicKeySize {+ return false+ }+ issuance = []ed25519.PublicKey{ed25519.PublicKey(enrollmentKey)}+ }+ n.keysMu.RLock()+ defer n.keysMu.RUnlock()+ for _, tk := range n.trustedKeys {+ if !containsPublicKey(issuance, tk.Key) {+ return true+ }+ }+ return false+}++func containsPublicKey(keys []ed25519.PublicKey, key ed25519.PublicKey) bool {+ for _, candidate := range keys {+ if candidate.Equal(key) {+ return true+ }+ }+ return false+}++// recordIdentityKeySet pins the trusted set to the identity just adopted.+func (n *SamNode) recordIdentityKeySet() {+ if n.Store == nil {+ return+ }+ n.keysMu.RLock()+ keys := publicKeysOf(n.trustedKeys)+ n.keysMu.RUnlock()+ if err := n.Store.SaveIdentityKeySet(keys); err != nil {+ logger.Errorf("Failed to persist the identity's key set: %v", err)+ }}// syncMeshInfo reads /info. The router addresses are persisted for the nextdiff --git a/internal/node/controlplane_test.go b/internal/node/controlplane_test.goindex ed5fc40a..97b491c7 100644--- a/internal/node/controlplane_test.go+++ b/internal/node/controlplane_test.go@@ -135,10 +135,8 @@ func TestSyncTrustedKeys(t *testing.T) {srv := keysServer(t, []ed25519.PublicKey{retiredPub, currentPub}, []ed25519.PrivateKey{retiredPriv, currentPriv})defer srv.Close()n := newNode(t)- if addedKeys, err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {t.Fatalf("syncTrustedKeys: %v", err)- } else if !addedKeys {- t.Fatal("syncTrustedKeys must report the newly learned key")}if len(n.trustedKeys) != 2 || !containsTrustedKey(n.trustedKeys, retiredPub) || !containsTrustedKey(n.trustedKeys, currentPub) {t.Errorf("trust set = %d keys, want retired and current", len(n.trustedKeys))@@ -150,11 +148,8 @@ func TestSyncTrustedKeys(t *testing.T) {if len(stored) != 2 {t.Errorf("persisted %d keys, want 2", len(stored))}- if n.rotationRefreshPending.Load() {- t.Error("adopting keys alone must not queue a credential refresh")- }- if addedKeys, err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil || addedKeys {- t.Fatalf("unchanged key set: addedKeys = %v, err = %v", addedKeys, err)+ if n.identityPredatesRotation() {+ t.Error("a node without an identity has nothing to refresh")}})@@ -162,7 +157,7 @@ func TestSyncTrustedKeys(t *testing.T) {srv := keysServer(t, []ed25519.PublicKey{strangerPub}, []ed25519.PrivateKey{strangerPriv})defer srv.Close()n := newNode(t)- if _, err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {t.Fatal("a /keys answer signed only by an unknown key must not be adopted")}if len(n.trustedKeys) != 1 || !n.trustedKeys[0].Key.Equal(currentPub) {@@ -174,7 +169,7 @@ func TestSyncTrustedKeys(t *testing.T) {srv := keysServer(t, []ed25519.PublicKey{currentPub, strangerPub}, nil)defer srv.Close()n := newNode(t)- if _, err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {t.Fatal("an unsigned /keys answer must not be adopted")}if containsTrustedKey(n.trustedKeys, strangerPub) {@@ -187,7 +182,7 @@ func TestSyncTrustedKeys(t *testing.T) {defer srv.Close()n := newNode(t)n.trustedKeys = nil- if _, err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {+ if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {t.Fatal("with no trusted key there is nothing to verify /keys against")}})diff --git a/internal/node/enroll.go b/internal/node/enroll.goindex e496ae5f..6f93e3e5 100644--- a/internal/node/enroll.go+++ b/internal/node/enroll.go@@ -81,9 +81,10 @@ func (n *SamNode) Enroll(ctx context.Context, controlPlaneURL string, jwt string// control plane too old to serve /keys still enrolls with one key.func (n *SamNode) adoptEnrolledKeys(ctx context.Context, controlPlaneURL string, newest []byte) {n.addTrustedKey(ed25519.PublicKey(newest))- if _, err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {+ if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {logger.Warnf("Could not fetch the control plane's full key set after enrollment (continuing with the enrollment key only): %v", err)}+ n.recordIdentityKeySet()}// enrollHTTP performs the HTTP half of enrollment for an explicit peerdiff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.goindex ac9524e4..4e8628a5 100644--- a/internal/node/enroll_test.go+++ b/internal/node/enroll_test.go@@ -39,9 +39,13 @@ import ("google.golang.org/protobuf/proto")-func TestEnrollmentKeyAdoptionDoesNotQueueRefresh(t *testing.T) {+// Enrollment hands out a biscuit signed by the current key and then learns+// the full key set, grace key included. Neither step is a rotation the new+// biscuit predates; a rotation that follows is.+func TestEnrollmentPinsIdentityKeySet(t *testing.T) {currentPub, currentPriv := mustGenerateKey(t)gracePub, gracePriv := mustGenerateKey(t)+ rotatedPub, _ := mustGenerateKey(t)store, err := NewStore(t.TempDir())if err != nil {t.Fatal(err)@@ -78,16 +82,29 @@ func TestEnrollmentKeyAdoptionDoesNotQueueRefresh(t *testing.T) {if !bytes.Equal(node.GetIdentity(), credential) || len(node.trustedKeys) != 2 {t.Fatal("enrollment must adopt the credential and both trusted keys")}- if node.rotationRefreshPending.Load() {- t.Error("enrollment key adoption queued a redundant refresh")+ if node.identityPredatesRotation() {+ t.Error("a freshly enrolled identity predates no rotation")}if err := node.SyncControlPlane(context.Background()); err != nil {t.Fatalf("sync after enrollment: %v", err)}- node.rotationRefreshPending.Store(true)- node.adoptEnrolledKeys(context.Background(), server.URL, currentPub)- if !node.rotationRefreshPending.Load() {- t.Fatal("enrollment key adoption cleared a pending rotation refresh")+ pinned, err := store.LoadIdentityKeySet()+ if err != nil || len(pinned) != 2 {+ t.Fatalf("identity key set = %d keys, err %v; want the grace and current keys", len(pinned), err)+ }++ node.handleKeyRotationEvent(&api.MeshEvent{NewPublicKey: rotatedPub})+ if !node.identityPredatesRotation() {+ t.Fatal("a key learned after issuance must mark the identity for refresh")+ }+ restarted := &SamNode{Store: store}+ if stored, err := store.LoadTrustedKeys(); err != nil {+ t.Fatal(err)+ } else {+ restarted.trustedKeys = stored+ }+ if !restarted.identityPredatesRotation() {+ t.Fatal("the refresh decision must survive a restart")}}diff --git a/internal/node/node.go b/internal/node/node.goindex 4714308c..972011eb 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -186,10 +186,13 @@ type SamNode struct {currentRelays []peer.AddrInforeprovideTrigger chan struct{}controlPlaneSyncTrigger chan struct{}- rotationRefreshPending atomic.Bool- BiscuitTimeout time.Duration- cachedIdentity atomic.Value- logger *golog.ZapEventLogger+ // refreshMu serializes credential issuance: the control plane redeems+ // only the last biscuit it issued, so two refreshes in flight would+ // invalidate each other.+ refreshMu sync.Mutex+ BiscuitTimeout time.Duration+ cachedIdentity atomic.Value+ logger *golog.ZapEventLogger// metricsRegistry holds this node's state collector; see metricsHandler.metricsOnce sync.Once@@ -1111,6 +1114,9 @@ func (e *RefreshError) Error() string {// RefreshEnrollment trades the expiring biscuit token for a new one using a cryptographic challenge.func (n *SamNode) RefreshEnrollment(ctx context.Context) error {+ n.refreshMu.Lock()+ defer n.refreshMu.Unlock()+// 1. Fetch current biscuitcurrentBiscuit, err := n.Store.LoadIdentity()if err != nil {@@ -1226,6 +1232,7 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {if err := n.Store.SaveIdentityExpiration(refreshResp.ExpiresAt); err != nil {return fmt.Errorf("failed to save refreshed expiration: %w", err)}+ n.recordIdentityKeySet()return nil}@@ -1398,17 +1405,16 @@ func containsTrustedKey(keys []TrustedKey, key ed25519.PublicKey) bool {// addTrustedKey appends a control plane public key to the trust set (no-op// on duplicates) and persists the updated set so it survives restarts.-func (n *SamNode) addTrustedKey(key ed25519.PublicKey) bool {+func (n *SamNode) addTrustedKey(key ed25519.PublicKey) {n.keysMu.Lock()if containsTrustedKey(n.trustedKeys, key) {n.keysMu.Unlock()- return false+ return}n.trustedKeys = append(n.trustedKeys, TrustedKey{Key: key, ReceivedAt: time.Now()})snapshot := append([]TrustedKey(nil), n.trustedKeys...)n.keysMu.Unlock()n.persistTrustedKeys(snapshot)- return true}func (n *SamNode) persistTrustedKeys(keys []TrustedKey) {@@ -1426,9 +1432,7 @@ func (n *SamNode) handleKeyRotationEvent(event *api.MeshEvent) {return}logger.Infow("[Mesh Event] key rotation received", "event", meshEventKeyRotation, "key", fmt.Sprintf("%x", event.NewPublicKey))- if n.addTrustedKey(ed25519.PublicKey(event.NewPublicKey)) && len(n.GetIdentity()) > 0 {- n.rotationRefreshPending.Store(true)- }+ n.addTrustedKey(ed25519.PublicKey(event.NewPublicKey))n.triggerControlPlaneSync()}diff --git a/internal/node/node_test.go b/internal/node/node_test.goindex 1a9af9e7..a22bcaef 100644--- a/internal/node/node_test.go+++ b/internal/node/node_test.go@@ -202,8 +202,8 @@ func TestHandleKeyRotationEvent(t *testing.T) {if len(node.trustedKeys) != 1 {t.Errorf("Expected 1 trusted key, got %d", len(node.trustedKeys))}- if node.rotationRefreshPending.Load() {- t.Error("an unenrolled node must not queue a credential refresh")+ if node.identityPredatesRotation() {+ t.Error("an unenrolled node has nothing to refresh")}}diff --git a/internal/node/refresh_test.go b/internal/node/refresh_test.goindex 6e2ab5c8..38d61e5f 100644--- a/internal/node/refresh_test.go+++ b/internal/node/refresh_test.go@@ -19,9 +19,12 @@ import ("context""crypto/ed25519""crypto/rand"+ "encoding/base64""errors""net/http""net/http/httptest"+ "strings"+ "sync""sync/atomic""testing""time"@@ -86,6 +89,9 @@ func newRefreshHarness(t *testing.T, refresh http.HandlerFunc) *refreshHarness {if err := store.SaveIdentity(current); err != nil {t.Fatal(err)}+ if err := store.SaveIdentityKeySet([]ed25519.PublicKey{cpPub}); err != nil {+ t.Fatal(err)+ }mux := http.NewServeMux()mux.HandleFunc("/refresh", refresh)@@ -173,6 +179,121 @@ func TestRotationRefreshDuringControlPlaneSync(t *testing.T) {}}+// A rotation learned before a restart must still be acted on after it: the+// persisted key set is then unchanged, so the decision cannot hinge on+// noticing a new key. Identities issued by builds that recorded no key set+// fall back to the enrollment key stored in the mesh config.+func TestRotationRefreshSurvivesRestart(t *testing.T) {+ for _, legacy := range []bool{false, true} {+ name := "recorded key set"+ if legacy {+ name = "pre-upgrade identity"+ }+ t.Run(name, func(t *testing.T) {+ newPub, newPriv := mustGenerateKey(t)+ var fresh []byte+ var refreshes atomic.Int32+ refresh := func(w http.ResponseWriter, request *http.Request) {+ if refreshes.Add(1) == 1 {+ http.Error(w, "temporarily unavailable", http.StatusServiceUnavailable)+ return+ }+ writeRefreshResponse(t, w, fresh)+ }+ harness := newRefreshHarness(t, refresh)+ fresh = mintRoleBiscuit(t, newPriv, harness.peerID, api.RoleNode)+ store := harness.node.Store+ if legacy {+ if err := store.SaveIdentityKeySet(nil); err != nil {+ t.Fatal(err)+ }+ if err := store.SaveMeshConfig(harness.cpPub, nil); err != nil {+ t.Fatal(err)+ }+ }+ mux := http.NewServeMux()+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{harness.cpPub, newPub}, []ed25519.PrivateKey{harness.cpPriv, newPriv}))+ mux.HandleFunc("/refresh", refresh)+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{}))+ mux.HandleFunc("/policies", protoHandler(t, &api.PolicyConfigGetResponse{}))+ server := httptest.NewServer(mux)+ defer server.Close()+ if err := store.SaveControlPlaneURL(server.URL); err != nil {+ t.Fatal(err)+ }+ if err := store.SaveTrustedKeys([]TrustedKey{{Key: harness.cpPub, ReceivedAt: time.Now()}}); err != nil {+ t.Fatal(err)+ }++ if err := harness.node.SyncControlPlane(context.Background()); err == nil {+ t.Fatal("sync must report the failed credential refresh")+ }+ harness.storedIdentityUnchanged(t)++ // Restart: only what the store holds carries over.+ restarted, err := NewSamNode(Options{PrivKey: GetOrGenerateKey(store), Store: store, ControlPlanePubKey: harness.cpPub, RequiredRole: api.RoleNode, BiscuitTimeout: time.Second})+ if err != nil {+ t.Fatal(err)+ }+ if len(restarted.trustedKeys) != 2 {+ t.Fatalf("restarted node loaded %d trusted keys, want the persisted pair", len(restarted.trustedKeys))+ }+ if err := restarted.SyncControlPlane(context.Background()); err != nil {+ t.Fatalf("refresh after restart: %v", err)+ }+ if !bytes.Equal(restarted.GetIdentity(), fresh) {+ t.Fatal("restarted node kept a credential signed by the retiring key")+ }+ if err := restarted.SyncControlPlane(context.Background()); err != nil {+ t.Fatalf("settled sync: %v", err)+ }+ if got := refreshes.Load(); got != 2 {+ t.Fatalf("refresh attempts = %d, want one failure before and one success after the restart", got)+ }+ })+ }+}++// The control plane redeems only the last biscuit it issued, so the rotation+// sync and the expiry renewal loop must not refresh at the same time: the+// loser would present an already-rotated biscuit and be refused as a replay.+func TestConcurrentRefreshesAreSerialized(t *testing.T) {+ var mu sync.Mutex+ var lastIssued []byte+ var harness *refreshHarness+ harness = newRefreshHarness(t, func(w http.ResponseWriter, request *http.Request) {+ presented, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(request.Header.Get("Authorization"), "Bearer "))+ if err != nil {+ t.Error(err)+ return+ }+ mu.Lock()+ defer mu.Unlock()+ if !bytes.Equal(presented, lastIssued) {+ http.Error(w, "Biscuit already rotated", http.StatusUnauthorized)+ return+ }+ lastIssued = mintRoleBiscuit(t, harness.cpPriv, harness.peerID, api.RoleNode)+ writeRefreshResponse(t, w, lastIssued)+ })+ lastIssued = harness.identity++ errs := make(chan error, 2)+ for range 2 {+ go func() { errs <- harness.node.RefreshEnrollment(context.Background()) }()+ }+ for range 2 {+ if err := <-errs; err != nil {+ t.Fatalf("concurrent refresh: %v", err)+ }+ }+ mu.Lock()+ defer mu.Unlock()+ if !bytes.Equal(harness.node.GetIdentity(), lastIssued) {+ t.Fatal("node does not hold the last biscuit the control plane issued")+ }+}+func (h *refreshHarness) storedIdentityUnchanged(t *testing.T) {t.Helper()stored, err := h.node.Store.LoadIdentity()diff --git a/internal/node/store.go b/internal/node/store.goindex 4d039d9a..12a9b158 100644--- a/internal/node/store.go+++ b/internal/node/store.go@@ -15,6 +15,7 @@package nodeimport (+ "crypto/ed25519""encoding/json""errors""fmt"@@ -39,6 +40,7 @@ const (keyOidcClientID = "oidc_client_id"keyOidcAudience = "oidc_audience"keyTrustedKeys = "trusted_keys"+ keyIdentityKeys = "identity_key_set")type Store struct {@@ -263,6 +265,35 @@ func (s *Store) LoadTrustedKeys() ([]TrustedKey, error) {return keys, err}+// SaveIdentityKeySet records the control plane public keys known when the+// stored identity was issued. A key that later appears outside this set is+// a rotation the identity predates, which is why it survives restarts.+func (s *Store) SaveIdentityKeySet(keys []ed25519.PublicKey) error {+ data, err := json.Marshal(keys)+ if err != nil {+ return err+ }+ return s.db.Update(func(tx *bbolt.Tx) error {+ b := tx.Bucket([]byte(bucketIdentity))+ return b.Put([]byte(keyIdentityKeys), data)+ })+}++// LoadIdentityKeySet returns the set saved by SaveIdentityKeySet, or nil when+// the identity was issued by a build that did not record it.+func (s *Store) LoadIdentityKeySet() ([]ed25519.PublicKey, error) {+ var keys []ed25519.PublicKey+ err := s.db.View(func(tx *bbolt.Tx) error {+ b := tx.Bucket([]byte(bucketIdentity))+ data := b.Get([]byte(keyIdentityKeys))+ if len(data) == 0 {+ return nil+ }+ return json.Unmarshal(data, &keys)+ })+ return keys, err+}+func (s *Store) LoadControlPlaneURL() (string, error) {var val []byteerr := s.db.View(func(tx *bbolt.Tx) error {@@ -288,6 +319,7 @@ func (s *Store) ResetMeshIdentity() error {keyOidcClientID,keyOidcAudience,keyTrustedKeys,+ keyIdentityKeys,"control_plane_public_key","router_addresses","control_plane_url",diff --git a/internal/router/metrics.go b/internal/router/metrics.goindex ee50a481..b5ca189f 100644--- a/internal/router/metrics.go+++ b/internal/router/metrics.go@@ -140,7 +140,7 @@ func (c *routerStateCollector) Collect(ch chan<- prometheus.Metric) {ch <- prometheus.MustNewConstMetric(c.dhtDesc, prometheus.GaugeValue, float64(r.DHT.RoutingTable().Size()))r.keysMu.RLock()- expiry := r.biscuitExpiration+ expiry := r.credential.expirationr.keysMu.RUnlock()if !expiry.IsZero() {ch <- prometheus.MustNewConstMetric(c.biscuitExpiryDesc, prometheus.GaugeValue, float64(expiry.Unix()))diff --git a/internal/router/metrics_test.go b/internal/router/metrics_test.goindex 3e17ecfd..3b0f93a9 100644--- a/internal/router/metrics_test.go+++ b/internal/router/metrics_test.go@@ -88,7 +88,7 @@ func TestRouterMetricsListener(t *testing.T) {r.bannedPeers.Store(peer.ID("peer-x"), true)expiry := time.Now().Add(time.Hour).Truncate(time.Second)r.keysMu.Lock()- r.biscuitExpiration = expiry+ r.credential.expiration = expiryr.keysMu.Unlock()r.isReady.Store(true)diff --git a/internal/router/router.go b/internal/router/router.goindex 702d9e43..87d35778 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -110,6 +110,54 @@ func (a *relayACL) AllowConnect(src peer.ID, srcAddr multiaddr.Multiaddr, dest preturn ok}+// credential is the router's own mesh identity as the control plane issued+// it. issuedUnder is the trusted key set at issuance: a trusted key outside+// it is a rotation the biscuit predates, so the biscuit must be refreshed+// before that key's predecessor leaves its grace period. It is nil until the+// first /keys sync after enrollment pins it, since enrollment hands out one+// key and the sync learns the rest.+type credential struct {+ biscuit []byte+ expiration time.Time+ issuedUnder []ed25519.PublicKey+}++func newCredential(biscuit []byte, expiresAt int64, issuedUnder []ed25519.PublicKey) credential {+ return credential{+ biscuit: biscuit,+ expiration: time.Unix(expiresAt, 0),+ issuedUnder: append([]ed25519.PublicKey(nil), issuedUnder...),+ }+}++// pin records trusted as the issuance set of a biscuit that has none yet.+func (c *credential) pin(trusted []ed25519.PublicKey) {+ if len(c.biscuit) > 0 && c.issuedUnder == nil {+ c.issuedUnder = append([]ed25519.PublicKey(nil), trusted...)+ }+}++// predatesRotation reports whether trusted holds a key unknown at issuance.+func (c credential) predatesRotation(trusted []ed25519.PublicKey) bool {+ return len(c.biscuit) > 0 && c.issuedUnder != nil && !subsetOf(trusted, c.issuedUnder)+}++func subsetOf(keys, set []ed25519.PublicKey) bool {+ for _, key := range keys {+ known := false+ for _, candidate := range set {+ if key.Equal(candidate) {+ known = true+ break+ }+ }+ if !known {+ return false+ }+ }+ return true+}+// Router represents a libp2p bootstrap/relay node.type Router struct {config Options@@ -123,16 +171,19 @@ type Router struct {// can open those streams.handshakeLimiter *ratelimit.PeerRateLimiter- // Keys & Identity- biscuitToken []byte- biscuitExpiration time.Time+ // trustedPublicKeys is what the control plane signs with now, the set+ // every peer credential is verified against; credential is this+ // router's own. Both are guarded by keysMu.trustedPublicKeys []ed25519.PublicKey+ credential credentialkeysMu sync.RWMutexenrollMu sync.MutexprivKey crypto.PrivKey+ // refreshMu serializes credential issuance: the control plane redeems+ // only the last biscuit it issued.+ refreshMu sync.Mutex- keysSyncTrigger chan struct{}- rotationRefreshPending bool+ keysSyncTrigger chan struct{}// Control contextsctx context.Context@@ -459,8 +510,7 @@ func (r *Router) enroll(peerID peer.ID) error {}r.keysMu.Lock()- r.biscuitToken = enrollResp.BiscuitToken- r.biscuitExpiration = time.Unix(enrollResp.Expiration, 0)+ r.credential = newCredential(enrollResp.BiscuitToken, enrollResp.Expiration, nil)r.trustedPublicKeys = []ed25519.PublicKey{enrollResp.ControlPlanePublicKey}r.keysMu.Unlock()@@ -604,8 +654,7 @@ func (r *Router) enrollBootstrap(peerID peer.ID) error {}r.keysMu.Lock()- r.biscuitToken = enrollResp.BiscuitToken- r.biscuitExpiration = time.Unix(enrollResp.Expiration, 0)+ r.credential = newCredential(enrollResp.BiscuitToken, enrollResp.Expiration, nil)r.trustedPublicKeys = []ed25519.PublicKey{enrollResp.ControlPlanePublicKey}r.keysMu.Unlock()@@ -692,26 +741,15 @@ func (r *Router) syncKeys() error {}r.keysMu.Lock()- for _, newKey := range newKeys {- known := false- for _, oldKey := range r.trustedPublicKeys {- if newKey.Equal(oldKey) {- known = true- break- }- }- if !known && len(r.biscuitToken) > 0 {- r.rotationRefreshPending = true- }- }r.trustedPublicKeys = newKeys+ r.credential.pin(newKeys)+ refresh := r.credential.predatesRotation(newKeys)r.keysMu.Unlock()- if r.rotationRefreshPending {+ if refresh {if err := r.RefreshEnrollment(r.ctx); err != nil {return fmt.Errorf("refresh enrollment after key rotation: %w", err)}- r.rotationRefreshPending = false}logger.Debugf("Synced %d valid public keys from control plane", len(newKeys))@@ -872,7 +910,7 @@ func (r *Router) runLeaseRenewalLoop() {func (r *Router) renewLease() {for attempt := 0; attempt < 2; attempt++ {r.keysMu.RLock()- biscuit := r.biscuitToken+ biscuit := r.credential.biscuitr.keysMu.RUnlock()if len(biscuit) == 0 {@@ -1190,7 +1228,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {// Send mutual response (our biscuit)r.keysMu.RLock()- ourBiscuit := r.biscuitToken+ ourBiscuit := r.credential.biscuitr.keysMu.RUnlock()writer := msgio.NewVarintWriter(s)@@ -1213,8 +1251,11 @@ func (r *Router) performMutualAuth(s network.Stream) error {_ = s.SetDeadline(time.Now().Add(5 * time.Second))// Send our biscuit+ r.keysMu.RLock()+ ourBiscuit := r.credential.biscuit+ r.keysMu.RUnlock()writer := msgio.NewVarintWriter(s)- authFrame := &api.AuthFrame{Biscuit: r.biscuitToken}+ authFrame := &api.AuthFrame{Biscuit: ourBiscuit}data, _ := proto.Marshal(authFrame)if err := writer.WriteMsg(data); err != nil {return fmt.Errorf("write mutual auth frame: %w", err)@@ -1328,8 +1369,11 @@ func isLoopbackOrLinkLocal(addr multiaddr.Multiaddr) bool {// RefreshEnrollment trades the expiring biscuit token for a new one using a cryptographic challenge.func (r *Router) RefreshEnrollment(ctx context.Context) error {+ r.refreshMu.Lock()+ defer r.refreshMu.Unlock()+r.keysMu.RLock()- currentBiscuit := r.biscuitToken+ currentBiscuit := r.credential.biscuitr.keysMu.RUnlock()if len(currentBiscuit) == 0 {@@ -1418,8 +1462,7 @@ func (r *Router) RefreshEnrollment(ctx context.Context) error {// Update local biscuit token and expiration under lockr.keysMu.Lock()- r.biscuitToken = refreshResp.BiscuitToken- r.biscuitExpiration = time.Unix(refreshResp.ExpiresAt, 0)+ r.credential = newCredential(refreshResp.BiscuitToken, refreshResp.ExpiresAt, r.trustedPublicKeys)r.keysMu.Unlock()logger.Infof("Router biscuit token refreshed successfully.")@@ -1447,7 +1490,7 @@ func (r *Router) runBiscuitRenewalLoop() {select {case <-ticker.C:r.keysMu.RLock()- expiration := r.biscuitExpiration+ expiration := r.credential.expirationr.keysMu.RUnlock()if !expiration.IsZero() && time.Until(expiration) < api.BiscuitTokenTTL/5 { // 80% elapsed lifespan of 24h (remaining time < 20% of TTL)diff --git a/internal/router/router_test.go b/internal/router/router_test.goindex eec83e02..e817c825 100644--- a/internal/router/router_test.go+++ b/internal/router/router_test.go@@ -452,8 +452,8 @@ func TestRouterProactiveTokenRefresh(t *testing.T) {defer func() { _ = r.Close() }()// Capture initial token and expiration- initialToken := r.biscuitToken- initialExpiration := r.biscuitExpiration+ initialToken := r.credential.biscuit+ initialExpiration := r.credential.expirationif len(initialToken) == 0 {t.Fatal("initial router biscuit token is empty")@@ -466,11 +466,11 @@ func TestRouterProactiveTokenRefresh(t *testing.T) {}// Assert that token and expiration updated- if bytes.Equal(r.biscuitToken, initialToken) {+ if bytes.Equal(r.credential.biscuit, initialToken) {t.Error("biscuit token did not change after refresh")}- if !r.biscuitExpiration.After(initialExpiration) {- t.Errorf("expected refreshed expiration %v to be after initial expiration %v", r.biscuitExpiration, initialExpiration)+ if !r.credential.expiration.After(initialExpiration) {+ t.Errorf("expected refreshed expiration %v to be after initial expiration %v", r.credential.expiration, initialExpiration)}}@@ -514,19 +514,19 @@ func TestRouterLeaseRenewalReEnrollOn401(t *testing.T) {// Corrupt router's biscuit token to simulate key rotation/expirationr.keysMu.Lock()- r.biscuitToken = []byte("invalid-corrupted-biscuit")+ r.credential.biscuit = []byte("invalid-corrupted-biscuit")r.keysMu.Unlock()// Call renewLease which should fail with 401, trigger reEnroll, and succeedr.renewLease()- // Verify that biscuitToken was replaced with a valid non-corrupted token+ // Verify that the biscuit was replaced with a valid non-corrupted tokenr.keysMu.RLock()- currentToken := r.biscuitToken+ currentToken := r.credential.biscuitr.keysMu.RUnlock()if bytes.Equal(currentToken, []byte("invalid-corrupted-biscuit")) {- t.Error("expected biscuitToken to be updated after 401 re-enrollment, but it remained corrupted")+ t.Error("expected the biscuit to be updated after 401 re-enrollment, but it remained corrupted")}}@@ -545,9 +545,9 @@ func TestRouterLeaseRenewalRepeated401Terminates(t *testing.T) {defer func() { _ = h.Close() }()r := &Router{- Host: h,- privKey: h.Peerstore().PrivKey(h.ID()),- biscuitToken: []byte("dummy-biscuit"),+ Host: h,+ privKey: h.Peerstore().PrivKey(h.ID()),+ credential: credential{biscuit: []byte("dummy-biscuit")},config: Options{ControlPlaneURL: ts.URL,},@@ -612,7 +612,7 @@ func TestRouterProactiveRefreshReEnrollOn401(t *testing.T) {// Corrupt router's biscuit token to simulate key rotationr.keysMu.Lock()- r.biscuitToken = []byte("stale-biscuit-signed-by-purged-key")+ r.credential.biscuit = []byte("stale-biscuit-signed-by-purged-key")r.keysMu.Unlock()// Trigger proactive refresh which should fail with 401, trigger reEnroll, and succeed@@ -621,13 +621,13 @@ func TestRouterProactiveRefreshReEnrollOn401(t *testing.T) {t.Fatalf("expected RefreshEnrollment to recover via reEnroll, but got error: %v", err)}- // Verify that biscuitToken was replaced with a valid fresh token+ // Verify that the biscuit was replaced with a valid fresh tokenr.keysMu.RLock()- currentToken := r.biscuitToken+ currentToken := r.credential.biscuitr.keysMu.RUnlock()if bytes.Equal(currentToken, []byte("stale-biscuit-signed-by-purged-key")) {- t.Error("expected biscuitToken to be updated after 401 refresh fallback, but it remained stale")+ t.Error("expected the biscuit to be updated after 401 refresh fallback, but it remained stale")}}@@ -872,7 +872,7 @@ func TestPerformMutualAuth(t *testing.T) {r := &Router{Host: clientHost,- biscuitToken: []byte("client-biscuit"),+ credential: credential{biscuit: []byte("client-biscuit")},trustedPublicKeys: []ed25519.PublicKey{oldPub, newPub},config: Options{BiscuitTimeout: time.Second,@@ -936,7 +936,7 @@ func TestHandleAuthHandshakeBoundsUnauthenticatedPeers(t *testing.T) {t.Fatal(err)}r.Host = serverHost- r.biscuitToken = []byte("router-biscuit")+ r.credential.biscuit = []byte("router-biscuit")r.trustedPublicKeys = []ed25519.PublicKey{cpPub}serverHost.SetStreamHandler(api.AuthProtocolID, r.HandleAuthHandshake)if err := clientHost.Connect(ctx, peer.AddrInfo{ID: serverHost.ID(), Addrs: serverHost.Addrs()}); err != nil {diff --git a/internal/router/trust_test.go b/internal/router/trust_test.goindex 3f12bef2..d4eab4fd 100644--- a/internal/router/trust_test.go+++ b/internal/router/trust_test.go@@ -19,10 +19,12 @@ import ("context""crypto/ed25519""crypto/rand"+ "encoding/base64""errors""net/http""net/http/httptest""strings"+ "sync""sync/atomic""testing""time"@@ -274,7 +276,7 @@ func TestSyncKeysRefreshesOnRotation(t *testing.T) {router := &Router{ctx: context.Background(),privKey: priv,- biscuitToken: mintRouterBiscuit(t, oldPriv, peerID, api.RoleRouter),+ credential: credential{biscuit: mintRouterBiscuit(t, oldPriv, peerID, api.RoleRouter), issuedUnder: []ed25519.PublicKey{oldPub}},trustedPublicKeys: []ed25519.PublicKey{oldPub},config: Options{ControlPlaneURL: server.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},}@@ -284,7 +286,7 @@ func TestSyncKeysRefreshesOnRotation(t *testing.T) {if err := router.syncKeys(); err != nil {t.Fatalf("retry refresh: %v", err)}- if !bytes.Equal(router.biscuitToken, fresh) {+ if !bytes.Equal(router.credential.biscuit, fresh) {t.Fatal("router kept a credential signed by the retiring key")}if err := router.syncKeys(); err != nil {@@ -293,6 +295,85 @@ func TestSyncKeysRefreshesOnRotation(t *testing.T) {if got := refreshes.Load(); got != 2 {t.Fatalf("refresh attempts = %d, want one failure and one success", got)}++ // A freshly enrolled router knows one key; its first sync learns the+ // grace key too. That is not a rotation its new biscuit predates.+ enrolled := &Router{+ ctx: context.Background(),+ privKey: priv,+ credential: credential{biscuit: fresh},+ trustedPublicKeys: []ed25519.PublicKey{newPub},+ config: Options{ControlPlaneURL: server.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},+ }+ if err := enrolled.syncKeys(); err != nil {+ t.Fatalf("first sync after enrollment: %v", err)+ }+ if got := refreshes.Load(); got != 2 {+ t.Fatalf("first sync after enrollment refreshed (%d attempts total)", got)+ }+ if len(enrolled.credential.issuedUnder) != 2 {+ t.Fatalf("issuance set = %d keys, want the synced pair", len(enrolled.credential.issuedUnder))+ }+}++// The control plane redeems only the last biscuit it issued, so the keys+// loop, the renewal loop and lease recovery must not refresh concurrently.+func TestRouterConcurrentRefreshesAreSerialized(t *testing.T) {+ cpPub, cpPriv, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ priv, _, err := crypto.GenerateEd25519Key(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ peerID, err := peer.IDFromPrivateKey(priv)+ if err != nil {+ t.Fatal(err)+ }+ var mu sync.Mutex+ lastIssued := mintRouterBiscuit(t, cpPriv, peerID, api.RoleRouter)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {+ presented, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(request.Header.Get("Authorization"), "Bearer "))+ if err != nil {+ t.Error(err)+ return+ }+ mu.Lock()+ defer mu.Unlock()+ if !bytes.Equal(presented, lastIssued) {+ http.Error(w, "Biscuit already rotated", http.StatusUnauthorized)+ return+ }+ lastIssued = mintRouterBiscuit(t, cpPriv, peerID, api.RoleRouter)+ body, err := proto.Marshal(&api.TokenRefreshResponse{BiscuitToken: lastIssued, ExpiresAt: time.Now().Add(time.Hour).Unix()})+ if err != nil {+ t.Error(err)+ return+ }+ _, _ = w.Write(body)+ }))+ defer server.Close()+ router := &Router{+ privKey: priv,+ credential: credential{biscuit: lastIssued},+ trustedPublicKeys: []ed25519.PublicKey{cpPub},+ config: Options{ControlPlaneURL: server.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},+ }+ errs := make(chan error, 2)+ for range 2 {+ go func() { errs <- router.RefreshEnrollment(context.Background()) }()+ }+ for range 2 {+ if err := <-errs; err != nil {+ t.Fatalf("concurrent refresh: %v", err)+ }+ }+ mu.Lock()+ defer mu.Unlock()+ if !bytes.Equal(router.credential.biscuit, lastIssued) {+ t.Fatal("router does not hold the last biscuit the control plane issued")+ }}func mintRouterBiscuit(t *testing.T, priv ed25519.PrivateKey, peerID peer.ID, role string) []byte {@@ -353,7 +434,7 @@ func TestRouterRefreshEnrollmentHardening(t *testing.T) {newRouter := func() *Router {return &Router{privKey: priv,- biscuitToken: current,+ credential: credential{biscuit: current},trustedPublicKeys: []ed25519.PublicKey{cpPub},config: Options{ControlPlaneURL: srv.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},}@@ -375,7 +456,7 @@ func TestRouterRefreshEnrollmentHardening(t *testing.T) {if err == nil || !strings.Contains(err.Error(), "403") {t.Fatalf("RefreshEnrollment = %v, want a 403 error", err)}- if !bytes.Equal(r.biscuitToken, current) {+ if !bytes.Equal(r.credential.biscuit, current) {t.Error("the current biscuit must be kept on a refused refresh")}})@@ -391,7 +472,7 @@ func TestRouterRefreshEnrollmentHardening(t *testing.T) {if err := r.RefreshEnrollment(context.Background()); err == nil {t.Fatal("RefreshEnrollment adopted a token it must have refused")}- if !bytes.Equal(r.biscuitToken, current) {+ if !bytes.Equal(r.credential.biscuit, current) {t.Error("the current biscuit must be kept when the refreshed one is refused")}})@@ -404,7 +485,7 @@ func TestRouterRefreshEnrollmentHardening(t *testing.T) {if err := r.RefreshEnrollment(context.Background()); err != nil {t.Fatalf("RefreshEnrollment: %v", err)}- if !bytes.Equal(r.biscuitToken, fresh) {+ if !bytes.Equal(r.credential.biscuit, fresh) {t.Error("a valid refreshed token must replace the current one")}})