Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
auth: refresh node and router credentials after key rotation
Antonio Ojea2b4a4393f5df
diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.goindex ab375fbb..8874a826 100644--- a/internal/node/controlplane_sync.go+++ b/internal/node/controlplane_sync.go@@ -47,8 +47,13 @@ func (n *SamNode) SyncControlPlane(ctx context.Context) error {return errors.New("control plane URL not found in store")}var errs []error- if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {+ if addedKeys, err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {errs = append(errs, fmt.Errorf("keys: %w", err))+ } else if len(n.GetIdentity()) > 0 && (n.rotationRefreshPending.Swap(false) || addedKeys) {+ if err := n.RefreshEnrollment(ctx); err != nil {+ n.rotationRefreshPending.Store(true)+ errs = append(errs, fmt.Errorf("refresh enrollment after key rotation: %w", err))+ }}if err := n.syncMeshInfo(ctx, controlPlaneURL); err != nil {errs = append(errs, fmt.Errorf("info: %w", err))@@ -66,28 +71,34 @@ func (n *SamNode) SyncControlPlane(ctx context.Context) error {// about. Verification against the keys already trusted keeps whoever answers// the URL from becoming the trust root; an empty answer is a failure so the// set is never wiped.-func (n *SamNode) syncTrustedKeys(ctx context.Context, controlPlaneURL string) error {+func (n *SamNode) syncTrustedKeys(ctx context.Context, controlPlaneURL string) (bool, error) {n.keysMu.RLock()existing := append([]TrustedKey(nil), n.trustedKeys...)n.keysMu.RUnlock()if len(existing) == 0 {- return errors.New("no trusted control plane keys to verify /keys against")+ return false, errors.New("no trusted control plane keys to verify /keys against")}keys, err := FetchControlPlaneKeys(ctx, controlPlaneURL, publicKeysOf(existing))if err != nil {- return err+ return false, err}if len(keys) == 0 {- return errors.New("/keys returned no keys")+ return false, errors.New("/keys returned no keys")}merged := mergeTrustedKeys(existing, keys, time.Now())+ addedKeys := falsen.keysMu.Lock()+ for _, key := range keys {+ if !containsTrustedKey(n.trustedKeys, key) {+ addedKeys = true+ }+ }n.trustedKeys = mergedsnapshot := append([]TrustedKey(nil), merged...)n.keysMu.Unlock()n.persistTrustedKeys(snapshot)logger.Debugf("Synced %d valid control plane keys", len(merged))- return nil+ return addedKeys, nil}// syncMeshInfo reads /info. The router addresses are persisted for the nextdiff --git a/internal/node/controlplane_test.go b/internal/node/controlplane_test.goindex 2b0f72d4..ed5fc40a 100644--- a/internal/node/controlplane_test.go+++ b/internal/node/controlplane_test.go@@ -30,7 +30,6 @@ import ("github.com/google/sam/api"lru "github.com/hashicorp/golang-lru/v2"- "github.com/libp2p/go-libp2p/core/crypto""github.com/libp2p/go-libp2p/core/peer""github.com/multiformats/go-multiaddr""google.golang.org/protobuf/proto"@@ -136,8 +135,10 @@ func TestSyncTrustedKeys(t *testing.T) {srv := keysServer(t, []ed25519.PublicKey{retiredPub, currentPub}, []ed25519.PrivateKey{retiredPriv, currentPriv})defer srv.Close()n := newNode(t)- if err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {+ if addedKeys, err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {t.Fatalf("syncTrustedKeys: %v", err)+ } else if !addedKeys {+ t.Fatal("syncTrustedKeys must report the newly learned key")}if len(n.trustedKeys) != 2 || !containsTrustedKey(n.trustedKeys, retiredPub) || !containsTrustedKey(n.trustedKeys, currentPub) {t.Errorf("trust set = %d keys, want retired and current", len(n.trustedKeys))@@ -149,13 +150,19 @@ func TestSyncTrustedKeys(t *testing.T) {if len(stored) != 2 {t.Errorf("persisted %d keys, want 2", len(stored))}+ if n.rotationRefreshPending.Load() {+ t.Error("adopting keys alone must not queue a credential refresh")+ }+ if addedKeys, err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil || addedKeys {+ t.Fatalf("unchanged key set: addedKeys = %v, err = %v", addedKeys, err)+ }})t.Run("rejects a set no trusted key signed", func(t *testing.T) {srv := keysServer(t, []ed25519.PublicKey{strangerPub}, []ed25519.PrivateKey{strangerPriv})defer srv.Close()n := newNode(t)- if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {+ if _, err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {t.Fatal("a /keys answer signed only by an unknown key must not be adopted")}if len(n.trustedKeys) != 1 || !n.trustedKeys[0].Key.Equal(currentPub) {@@ -167,7 +174,7 @@ func TestSyncTrustedKeys(t *testing.T) {srv := keysServer(t, []ed25519.PublicKey{currentPub, strangerPub}, nil)defer srv.Close()n := newNode(t)- if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {+ if _, err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {t.Fatal("an unsigned /keys answer must not be adopted")}if containsTrustedKey(n.trustedKeys, strangerPub) {@@ -180,7 +187,7 @@ func TestSyncTrustedKeys(t *testing.T) {defer srv.Close()n := newNode(t)n.trustedKeys = nil- if err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {+ if _, err := n.syncTrustedKeys(context.Background(), srv.URL); err == nil {t.Fatal("with no trusted key there is nothing to verify /keys against")}})@@ -411,6 +418,21 @@ func TestSyncControlPlaneBeforeStart(t *testing.T) {mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{cpPub, gracePub}, []ed25519.PrivateKey{cpPriv, gracePriv}))mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{RouterAddresses: []string{freshRouter}}))mux.HandleFunc("/policies", protoHandler(t, &api.PolicyConfigGetResponse{}))+ mux.HandleFunc("/refresh", func(w http.ResponseWriter, request *http.Request) {+ body, err := io.ReadAll(request.Body)+ var refresh api.TokenRefreshRequest+ if err != nil || proto.Unmarshal(body, &refresh) != nil {+ t.Error("invalid refresh request")+ http.Error(w, "invalid request", http.StatusBadRequest)+ return+ }+ peerID, err := peer.Decode(refresh.PeerId)+ if err != nil {+ t.Error(err)+ return+ }+ writeRefreshResponse(t, w, mintRoleBiscuit(t, cpPriv, peerID, api.RoleNode))+ })srv := httptest.NewServer(mux)defer srv.Close()@@ -434,7 +456,8 @@ func TestSyncControlPlaneBeforeStart(t *testing.T) {if err != nil {t.Fatal(err)}- priv, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ priv := GetOrGenerateKey(store)+ peerID, err := peer.IDFromPrivateKey(priv)if err != nil {t.Fatal(err)}@@ -442,7 +465,11 @@ func TestSyncControlPlaneBeforeStart(t *testing.T) {if err != nil {t.Fatal(err)}- n.SetIdentityCache([]byte("identity"))+ identity := mintRoleBiscuit(t, cpPriv, peerID, api.RoleNode)+ if err := store.SaveIdentity(identity); err != nil {+ t.Fatal(err)+ }+ n.SetIdentityCache(identity)return n}diff --git a/internal/node/enroll.go b/internal/node/enroll.goindex c113a3bb..e496ae5f 100644--- a/internal/node/enroll.go+++ b/internal/node/enroll.go@@ -81,7 +81,7 @@ func (n *SamNode) Enroll(ctx context.Context, controlPlaneURL string, jwt string// control plane too old to serve /keys still enrolls with one key.func (n *SamNode) adoptEnrolledKeys(ctx context.Context, controlPlaneURL string, newest []byte) {n.addTrustedKey(ed25519.PublicKey(newest))- if err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {+ if _, err := n.syncTrustedKeys(ctx, controlPlaneURL); err != nil {logger.Warnf("Could not fetch the control plane's full key set after enrollment (continuing with the enrollment key only): %v", err)}}diff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.goindex 1facf273..ac9524e4 100644--- a/internal/node/enroll_test.go+++ b/internal/node/enroll_test.go@@ -39,6 +39,58 @@ import ("google.golang.org/protobuf/proto")+func TestEnrollmentKeyAdoptionDoesNotQueueRefresh(t *testing.T) {+ currentPub, currentPriv := mustGenerateKey(t)+ gracePub, gracePriv := mustGenerateKey(t)+ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = store.Close() })+ privateKey := GetOrGenerateKey(store)+ peerID, err := peer.IDFromPrivateKey(privateKey)+ if err != nil {+ t.Fatal(err)+ }+ credential := mintRoleBiscuit(t, currentPriv, peerID, api.RoleNode)+ mux := http.NewServeMux()+ mux.HandleFunc("/register", protoHandler(t, &api.EnrollResponse{+ BiscuitToken: credential,+ ControlPlanePublicKey: currentPub,+ Expiration: time.Now().Add(24 * time.Hour).Unix(),+ }))+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{gracePub, currentPub}, []ed25519.PrivateKey{gracePriv, currentPriv}))+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{}))+ mux.HandleFunc("/policies", protoHandler(t, &api.PolicyConfigGetResponse{}))+ mux.HandleFunc("/refresh", func(w http.ResponseWriter, request *http.Request) {+ t.Error("freshly enrolled credential must not be refreshed")+ http.Error(w, "unexpected refresh", http.StatusInternalServerError)+ })+ server := httptest.NewServer(mux)+ defer server.Close()+ if err := store.SaveControlPlaneURL(server.URL); err != nil {+ t.Fatal(err)+ }+ node := &SamNode{Store: store, BiscuitTimeout: time.Second, config: Options{RequiredRole: api.RoleNode}}+ if _, err := node.enrollHTTP(context.Background(), server.URL, "test-jwt", peerID, privateKey); err != nil {+ t.Fatal(err)+ }+ if !bytes.Equal(node.GetIdentity(), credential) || len(node.trustedKeys) != 2 {+ t.Fatal("enrollment must adopt the credential and both trusted keys")+ }+ if node.rotationRefreshPending.Load() {+ t.Error("enrollment key adoption queued a redundant refresh")+ }+ if err := node.SyncControlPlane(context.Background()); err != nil {+ t.Fatalf("sync after enrollment: %v", err)+ }+ node.rotationRefreshPending.Store(true)+ node.adoptEnrolledKeys(context.Background(), server.URL, currentPub)+ if !node.rotationRefreshPending.Load() {+ t.Fatal("enrollment key adoption cleared a pending rotation refresh")+ }+}+// startMockRouterWithKey runs a minimal router whose auth response biscuit is// signed by the given CP key, so nodes trusting that key can authenticate.func startMockRouterWithKey(t *testing.T, cpPriv ed25519.PrivateKey, role string) string {diff --git a/internal/node/node.go b/internal/node/node.goindex 6a2184a3..4714308c 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -186,6 +186,7 @@ type SamNode struct {currentRelays []peer.AddrInforeprovideTrigger chan struct{}controlPlaneSyncTrigger chan struct{}+ rotationRefreshPending atomic.BoolBiscuitTimeout time.DurationcachedIdentity atomic.Valuelogger *golog.ZapEventLogger@@ -1397,16 +1398,17 @@ func containsTrustedKey(keys []TrustedKey, key ed25519.PublicKey) bool {// addTrustedKey appends a control plane public key to the trust set (no-op// on duplicates) and persists the updated set so it survives restarts.-func (n *SamNode) addTrustedKey(key ed25519.PublicKey) {+func (n *SamNode) addTrustedKey(key ed25519.PublicKey) bool {n.keysMu.Lock()if containsTrustedKey(n.trustedKeys, key) {n.keysMu.Unlock()- return+ return false}n.trustedKeys = append(n.trustedKeys, TrustedKey{Key: key, ReceivedAt: time.Now()})snapshot := append([]TrustedKey(nil), n.trustedKeys...)n.keysMu.Unlock()n.persistTrustedKeys(snapshot)+ return true}func (n *SamNode) persistTrustedKeys(keys []TrustedKey) {@@ -1424,7 +1426,10 @@ func (n *SamNode) handleKeyRotationEvent(event *api.MeshEvent) {return}logger.Infow("[Mesh Event] key rotation received", "event", meshEventKeyRotation, "key", fmt.Sprintf("%x", event.NewPublicKey))- n.addTrustedKey(ed25519.PublicKey(event.NewPublicKey))+ if n.addTrustedKey(ed25519.PublicKey(event.NewPublicKey)) && len(n.GetIdentity()) > 0 {+ n.rotationRefreshPending.Store(true)+ }+ n.triggerControlPlaneSync()}// pruneTrustedKeys drops keys older than gracePeriod but always keeps thediff --git a/internal/node/node_test.go b/internal/node/node_test.goindex c86337ae..1a9af9e7 100644--- a/internal/node/node_test.go+++ b/internal/node/node_test.go@@ -202,6 +202,9 @@ func TestHandleKeyRotationEvent(t *testing.T) {if len(node.trustedKeys) != 1 {t.Errorf("Expected 1 trusted key, got %d", len(node.trustedKeys))}+ if node.rotationRefreshPending.Load() {+ t.Error("an unenrolled node must not queue a credential refresh")+ }}// TestKeyRotationEventSurvivesRestart covers the offline/restart half ofdiff --git a/internal/node/refresh_test.go b/internal/node/refresh_test.goindex 85dc31bb..6e2ab5c8 100644--- a/internal/node/refresh_test.go+++ b/internal/node/refresh_test.go@@ -22,6 +22,7 @@ import ("errors""net/http""net/http/httptest"+ "sync/atomic""testing""time"@@ -114,6 +115,64 @@ func writeRefreshResponse(t *testing.T, w http.ResponseWriter, token []byte) {_, _ = w.Write(data)}+func TestRotationRefreshDuringControlPlaneSync(t *testing.T) {+ for _, viaEvent := range []bool{false, true} {+ name := "periodic sync"+ if viaEvent {+ name = "rotation event"+ }+ t.Run(name, func(t *testing.T) {+ newPub, newPriv := mustGenerateKey(t)+ var fresh []byte+ var refreshes atomic.Int32+ refresh := func(w http.ResponseWriter, request *http.Request) {+ if refreshes.Add(1) == 1 {+ http.Error(w, "temporarily unavailable", http.StatusServiceUnavailable)+ return+ }+ writeRefreshResponse(t, w, fresh)+ }+ harness := newRefreshHarness(t, refresh)+ fresh = mintRoleBiscuit(t, newPriv, harness.peerID, api.RoleNode)+ mux := http.NewServeMux()+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{harness.cpPub, newPub}, []ed25519.PrivateKey{harness.cpPriv, newPriv}))+ mux.HandleFunc("/refresh", refresh)+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{}))+ mux.HandleFunc("/policies", protoHandler(t, &api.PolicyConfigGetResponse{}))+ server := httptest.NewServer(mux)+ defer server.Close()+ if err := harness.node.Store.SaveControlPlaneURL(server.URL); err != nil {+ t.Fatal(err)+ }+ if viaEvent {+ harness.node.controlPlaneSyncTrigger = make(chan struct{}, 1)+ harness.node.handleKeyRotationEvent(&api.MeshEvent{NewPublicKey: newPub})+ select {+ case <-harness.node.controlPlaneSyncTrigger:+ default:+ t.Fatal("rotation event did not trigger a control-plane sync")+ }+ }+ if err := harness.node.SyncControlPlane(context.Background()); err == nil {+ t.Fatal("sync must report the failed credential refresh")+ }+ harness.storedIdentityUnchanged(t)+ if err := harness.node.SyncControlPlane(context.Background()); err != nil {+ t.Fatalf("retry refresh: %v", err)+ }+ if !bytes.Equal(harness.node.GetIdentity(), fresh) {+ t.Fatal("node kept a credential signed by the retiring key")+ }+ if err := harness.node.SyncControlPlane(context.Background()); err != nil {+ t.Fatalf("unchanged keys: %v", err)+ }+ if got := refreshes.Load(); got != 2 {+ t.Fatalf("refresh attempts = %d, want one failure and one success", got)+ }+ })+ }+}+func (h *refreshHarness) storedIdentityUnchanged(t *testing.T) {t.Helper()stored, err := h.node.Store.LoadIdentity()diff --git a/internal/router/router.go b/internal/router/router.goindex 02c438b5..702d9e43 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -131,6 +131,9 @@ type Router struct {enrollMu sync.MutexprivKey crypto.PrivKey+ keysSyncTrigger chan struct{}+ rotationRefreshPending bool+// Control contextsctx context.Contextcancel context.CancelFunc@@ -162,6 +165,7 @@ func NewRouter(ctx context.Context, config Options) (*Router, error) {ctx: ctx,cancel: cancel,handshakeLimiter: handshakeLimiter,+ keysSyncTrigger: make(chan struct{}, 1),}, nil}@@ -688,9 +692,28 @@ func (r *Router) syncKeys() error {}r.keysMu.Lock()+ for _, newKey := range newKeys {+ known := false+ for _, oldKey := range r.trustedPublicKeys {+ if newKey.Equal(oldKey) {+ known = true+ break+ }+ }+ if !known && len(r.biscuitToken) > 0 {+ r.rotationRefreshPending = true+ }+ }r.trustedPublicKeys = newKeysr.keysMu.Unlock()+ if r.rotationRefreshPending {+ if err := r.RefreshEnrollment(r.ctx); err != nil {+ return fmt.Errorf("refresh enrollment after key rotation: %w", err)+ }+ r.rotationRefreshPending = false+ }+logger.Debugf("Synced %d valid public keys from control plane", len(newKeys))return nil}@@ -798,13 +821,18 @@ func (r *Router) listenForControlPlaneEvents(ctx context.Context) {}case api.MeshEvent_KEY_ROTATION:logger.Infof("[Router Event] Received KEY_ROTATION event, triggering key sync")- if err := r.syncKeys(); err != nil {- logger.Warnf("[Router Event] Failed to sync keys after KEY_ROTATION event: %v", err)- }+ r.triggerKeysSync()}}}+func (r *Router) triggerKeysSync() {+ select {+ case r.keysSyncTrigger <- struct{}{}:+ default:+ }+}+func (r *Router) runKeysSyncLoop() {defer r.wg.Done()ticker := time.NewTicker(r.config.KeysSyncInterval)@@ -813,12 +841,13 @@ func (r *Router) runKeysSyncLoop() {for {select {case <-ticker.C:- if err := r.syncKeys(); err != nil {- logger.Errorf("Failed periodic keys sync: %v", err)- }+ case <-r.keysSyncTrigger:case <-r.ctx.Done():return}+ if err := r.syncKeys(); err != nil {+ logger.Errorf("Failed keys sync: %v", err)+ }}}diff --git a/internal/router/trust_test.go b/internal/router/trust_test.goindex f0f6df7d..3f12bef2 100644--- a/internal/router/trust_test.go+++ b/internal/router/trust_test.go@@ -23,6 +23,7 @@ import ("net/http""net/http/httptest""strings"+ "sync/atomic""testing""time"@@ -121,6 +122,17 @@ func TestSyncKeysRequiresTrustedSignature(t *testing.T) {}})+ t.Run("empty set preserves trusted keys", func(t *testing.T) {+ body = marshal(&api.KeysResponse{Timestamp: time.Now().UnixMilli()})+ router := newRouter()+ if err := router.syncKeys(); err == nil {+ t.Fatal("an empty /keys answer must not be adopted")+ }+ if keys := router.getTrustedPublicKeys(); len(keys) != 1 || !keys[0].Equal(oldPub) {+ t.Errorf("trust set changed on an empty answer: %d keys", len(keys))+ }+ })+t.Run("unsigned set is refused", func(t *testing.T) {body = marshal(&api.KeysResponse{PublicKeys: [][]byte{oldPub, attackerPub}})r := newRouter()@@ -148,6 +160,141 @@ func TestSyncKeysRequiresTrustedSignature(t *testing.T) {})}+func TestKeysSyncLoopCoalescesTriggers(t *testing.T) {+ publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ keys := &api.KeysResponse{PublicKeys: [][]byte{publicKey}}+ if err := api.SignKeysResponse(keys, []ed25519.PrivateKey{privateKey}, time.Now()); err != nil {+ t.Fatal(err)+ }+ body, err := proto.Marshal(keys)+ if err != nil {+ t.Fatal(err)+ }+ requests := make(chan struct{}, 4)+ release := make(chan struct{})+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {+ requests <- struct{}{}+ select {+ case <-release:+ _, _ = w.Write(body)+ case <-request.Context().Done():+ }+ }))+ t.Cleanup(server.Close)+ router, err := NewRouter(context.Background(), Options{ControlPlaneURL: server.URL, KeysSyncInterval: time.Hour})+ if err != nil {+ t.Fatal(err)+ }+ router.trustedPublicKeys = []ed25519.PublicKey{publicKey}+ router.wg.Add(1)+ go router.runKeysSyncLoop()+ t.Cleanup(func() {+ router.cancel()+ router.wg.Wait()+ })+ await := func(signal <-chan struct{}, message string) {+ t.Helper()+ select {+ case <-signal:+ case <-time.After(2 * time.Second):+ t.Fatal(message)+ }+ }+ router.triggerKeysSync()+ await(requests, "key sync did not start")+ notified := make(chan struct{})+ go func() {+ for range 10 {+ router.triggerKeysSync()+ }+ router.getTrustedPublicKeys()+ close(notified)+ }()+ await(notified, "notifications or key readers blocked on the control-plane request")+ if pending := len(router.keysSyncTrigger); pending != 1 {+ t.Fatalf("pending triggers = %d, want one coalesced follow-up", pending)+ }+ release <- struct{}{}+ await(requests, "pending key sync was lost")+ if pending := len(router.keysSyncTrigger); pending != 0 {+ t.Fatalf("pending triggers = %d after follow-up", pending)+ }+}++func TestSyncKeysRefreshesOnRotation(t *testing.T) {+ oldPub, oldPriv, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ priv, _, err := crypto.GenerateEd25519Key(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ peerID, err := peer.IDFromPrivateKey(priv)+ if err != nil {+ t.Fatal(err)+ }+ fresh := mintRouterBiscuit(t, newPriv, peerID, api.RoleRouter)+ keys := &api.KeysResponse{PublicKeys: [][]byte{oldPub, newPub}}+ if err := api.SignKeysResponse(keys, []ed25519.PrivateKey{oldPriv, newPriv}, time.Now()); err != nil {+ t.Fatal(err)+ }+ var refreshes atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {+ var response proto.Message+ switch request.URL.Path {+ case "/keys":+ response = keys+ case "/refresh":+ if refreshes.Add(1) == 1 {+ http.Error(w, "temporarily unavailable", http.StatusServiceUnavailable)+ return+ }+ response = &api.TokenRefreshResponse{BiscuitToken: fresh, ExpiresAt: time.Now().Add(24 * time.Hour).Unix()}+ default:+ t.Errorf("unexpected request to %s", request.URL.Path)+ http.NotFound(w, request)+ return+ }+ body, marshalErr := proto.Marshal(response)+ if marshalErr != nil {+ t.Error(marshalErr)+ return+ }+ _, _ = w.Write(body)+ }))+ defer server.Close()+ router := &Router{+ ctx: context.Background(),+ privKey: priv,+ biscuitToken: mintRouterBiscuit(t, oldPriv, peerID, api.RoleRouter),+ trustedPublicKeys: []ed25519.PublicKey{oldPub},+ config: Options{ControlPlaneURL: server.URL, RequiredRole: api.RoleRouter, BiscuitTimeout: time.Second},+ }+ if err := router.syncKeys(); err == nil {+ t.Fatal("rotation sync must report the failed credential refresh")+ }+ if err := router.syncKeys(); err != nil {+ t.Fatalf("retry refresh: %v", err)+ }+ if !bytes.Equal(router.biscuitToken, fresh) {+ t.Fatal("router kept a credential signed by the retiring key")+ }+ if err := router.syncKeys(); err != nil {+ t.Fatalf("unchanged keys: %v", err)+ }+ if got := refreshes.Load(); got != 2 {+ t.Fatalf("refresh attempts = %d, want one failure and one success", got)+ }+}+func mintRouterBiscuit(t *testing.T, priv ed25519.PrivateKey, peerID peer.ID, role string) []byte {t.Helper()builder := biscuit.NewBuilder(priv)