Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
fix: control plane - mint a fresh biscuit when an approved enrollment's token has expired
Fernando Marino1d47598ccd6a
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex e3957615..1defbc6d 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -1305,7 +1305,13 @@ func (s *Server) HandleEnroll(w http.ResponseWriter, r *http.Request) {// Request already exists, return statusvar resp *api.BootstrapEnrollResponseif existingReq.Status == api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {- resp, err = s.buildApprovedBootstrapEnrollResponse(ctx, existingReq.BiscuitToken, existingReq.ResolvedAt)+ biscuitToken, resolvedAt, refreshErr := s.refreshExpiredBootstrapBiscuit(ctx, existingReq)+ if refreshErr != nil {+ logger.Errorf("Failed to refresh expired enrollment biscuit: %v", refreshErr)+ http.Error(w, "Internal server error", http.StatusInternalServerError)+ return+ }+ resp, err = s.buildApprovedBootstrapEnrollResponse(ctx, biscuitToken, resolvedAt)if err != nil {logger.Errorf("Failed to build approved response: %v", err)http.Error(w, "Internal server error", http.StatusInternalServerError)@@ -1516,7 +1522,13 @@ func (s *Server) HandleEnrollStatus(w http.ResponseWriter, r *http.Request) {var resp *api.BootstrapEnrollResponseif enrollReq.Status == api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {- resp, err = s.buildApprovedBootstrapEnrollResponse(ctx, enrollReq.BiscuitToken, enrollReq.ResolvedAt)+ biscuitToken, resolvedAt, refreshErr := s.refreshExpiredBootstrapBiscuit(ctx, enrollReq)+ if refreshErr != nil {+ logger.Errorf("Failed to refresh expired enrollment biscuit: %v", refreshErr)+ http.Error(w, "Internal server error", http.StatusInternalServerError)+ return+ }+ resp, err = s.buildApprovedBootstrapEnrollResponse(ctx, biscuitToken, resolvedAt)if err != nil {logger.Errorf("Failed to build approved response: %v", err)http.Error(w, "Internal server error", http.StatusInternalServerError)@@ -1917,6 +1929,69 @@ func (s *Server) HandleAdminRevoke(w http.ResponseWriter, r *http.Request) {_, _ = w.Write(respData)}+// refreshExpiredBootstrapBiscuit returns an already-approved enrollment+// request's biscuit unchanged if it is still within its TTL, or mints and+// persists a fresh one otherwise.+//+// Bootstrap peers (routers and other non-OIDC nodes) only ever call /refresh+// proactively while already running; they have no equivalent of a stored+// refresh token to fall back on across a restart. If a peer's data directory+// is idle for longer than BiscuitTTL and then restarts, its old biscuit is+// already expired by the time it re-enrolls, and re-enrolling just hits the+// "existing enrollment request" branch below, which used to hand back that+// same expired token every time -- an approve-fails-approve-fails loop with+// no way out short of an operator manually deleting the enrollment record.+func (s *Server) refreshExpiredBootstrapBiscuit(ctx context.Context, existingReq *storage.EnrollmentRequest) ([]byte, *time.Time, error) {+ if existingReq.ResolvedAt != nil && existingReq.ResolvedAt.Add(s.config.BiscuitTTL).After(time.Now()) {+ return existingReq.BiscuitToken, existingReq.ResolvedAt, nil+ }++ pID, err := peer.Decode(existingReq.PeerID)+ if err != nil {+ return nil, nil, fmt.Errorf("invalid stored peer id %q: %w", existingReq.PeerID, err)+ }++ // The enrolled node record (not the bootstrap token, which may since have+ // been consumed or pruned) is the durable source of the role and labels+ // to re-mint -- both approval paths write it before ever returning this+ // enrollment request as APPROVED.+ nodeRecord, err := s.store.GetNode(ctx, existingReq.PeerID)+ if err != nil {+ return nil, nil, fmt.Errorf("failed to retrieve enrolled node %s: %w", existingReq.PeerID, err)+ }++ privKey, _, err := s.store.GetCurrentKey(ctx)+ if err != nil {+ return nil, nil, fmt.Errorf("failed to retrieve signing key: %w", err)+ }++ policyRoles, _, err := s.store.GetMeshPolicy(ctx)+ if err != nil && err != storage.ErrNotFound {+ return nil, nil, fmt.Errorf("failed to retrieve mesh policy: %w", err)+ }++ biscuitExpiry := time.Now().Add(s.config.BiscuitTTL)+ biscuitBytes, err := identity.MintBootstrapBiscuitToken(privKey, pID, nodeRecord.Role, biscuitExpiry, policyRoles, nodeRecord.Labels)+ if err != nil {+ return nil, nil, fmt.Errorf("failed to mint refreshed bootstrap biscuit: %w", err)+ }++ if err := s.store.UpdateEnrollmentRequest(ctx, existingReq.ID, api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED, biscuitBytes, existingReq.ResolvedBy); err != nil {+ return nil, nil, fmt.Errorf("failed to persist refreshed enrollment request: %w", err)+ }++ // Keep the node record's biscuit in lockstep: /refresh's reuse-detection+ // compares a presented biscuit against exactly this field.+ nodeRecord.Biscuit = biscuitBytes+ nodeRecord.EnrolledAt = time.Now()+ if err := s.store.EnrollNode(ctx, nodeRecord); err != nil {+ return nil, nil, fmt.Errorf("failed to persist refreshed node record: %w", err)+ }++ resolvedAt := time.Now()+ return biscuitBytes, &resolvedAt, nil+}+func (s *Server) buildApprovedBootstrapEnrollResponse(ctx context.Context, biscuitToken []byte, resolvedAt *time.Time) (*api.BootstrapEnrollResponse, error) {_, pubKey, err := s.store.GetCurrentKey(ctx)if err != nil {diff --git a/internal/controlplane/stale_biscuit_replay_test.go b/internal/controlplane/stale_biscuit_replay_test.gonew file mode 100644index 00000000..a27524ac--- /dev/null+++ b/internal/controlplane/stale_biscuit_replay_test.go@@ -0,0 +1,188 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package controlplane++import (+ "bytes"+ "crypto/ed25519"+ "encoding/json"+ "io"+ "net/http"+ "testing"+ "time"++ "github.com/google/sam/api"+ "github.com/google/sam/internal/identity"+ "github.com/libp2p/go-libp2p/core/crypto"+ "github.com/libp2p/go-libp2p/core/peer"+ "google.golang.org/protobuf/proto"+)++// TestReEnrollAfterExpiryMintsFreshBiscuit pins the fix for the bootstrap+// enrollment stale-biscuit-replay bug: a peer whose approved enrollment+// record outlives BiscuitTTL (e.g. a router restarting after several idle+// days) and re-submits /enroll or polls /enroll/status must get back a+// biscuit that actually verifies, not the original, now-expired one.+//+// Before the fix, both endpoints' "existing enrollment request" branch+// returned existingReq.BiscuitToken verbatim forever, so a router hitting a+// 401 on lease renewal and re-enrolling would receive the exact same expired+// token every time -- an unbreakable enroll -> reject -> re-enroll loop,+// identical to the one HandleRouterLease logs as "failed biscuit+// verification: ... $time <= $exp".+func TestReEnrollAfterExpiryMintsFreshBiscuit(t *testing.T) {+ issuer, _ := startCustomMockOIDC(t)+ srv, store, baseURL := setupTestServer(t, issuer, func(o *Options) {+ o.BiscuitTTL = 1 * time.Second+ })+ defer func() {+ _ = srv.Close()+ _ = store.Close()+ }()+ srv.config.AdminToken = "super-secret-admin-token"+ srv.config.AutoApproveEnrollment = true++ client := &http.Client{Timeout: 5 * time.Second}++ adminReqBody := []byte(`{"role": "sam:role:router", "ttl_hours": 2, "max_usages": 5, "description": "stale replay test"}`)+ req, err := http.NewRequest("POST", baseURL+"/admin/bootstrap-tokens", bytes.NewBuffer(adminReqBody))+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", "Bearer super-secret-admin-token")+ resp, err := client.Do(req)+ if err != nil {+ t.Fatalf("failed to create bootstrap token: %v", err)+ }+ var tokenDetails struct {+ Token string `json:"token"`+ }+ _ = json.NewDecoder(resp.Body).Decode(&tokenDetails)+ _ = resp.Body.Close()+ if tokenDetails.Token == "" {+ t.Fatal("empty bootstrap token")+ }++ priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ if err != nil {+ t.Fatal(err)+ }+ pID, err := peer.IDFromPrivateKey(priv)+ if err != nil {+ t.Fatal(err)+ }+ pubBytes, err := crypto.MarshalPublicKey(pub)+ if err != nil {+ t.Fatal(err)+ }++ doEnroll := func() *api.BootstrapEnrollResponse {+ t.Helper()+ ts, sig := enrollPoP(t, priv, pID.String())+ data, err := proto.Marshal(&api.BootstrapEnrollRequest{+ BootstrapToken: tokenDetails.Token,+ PeerId: pID.String(),+ PublicKey: pubBytes,+ RequestedRole: api.RoleRouter,+ Timestamp: ts,+ ChallengeSignature: sig,+ })+ if err != nil {+ t.Fatal(err)+ }+ resp, err := client.Post(baseURL+"/enroll", "application/x-protobuf", bytes.NewReader(data))+ if err != nil {+ t.Fatalf("/enroll failed: %v", err)+ }+ defer func() { _ = resp.Body.Close() }()+ body, _ := io.ReadAll(resp.Body)+ if resp.StatusCode != http.StatusOK {+ t.Fatalf("/enroll status %s: %s", resp.Status, string(body))+ }+ var out api.BootstrapEnrollResponse+ if err := proto.Unmarshal(body, &out); err != nil {+ t.Fatalf("unmarshal BootstrapEnrollResponse: %v", err)+ }+ return &out+ }++ first := doEnroll()+ if first.Status != api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {+ t.Fatalf("expected APPROVED on first enroll, got %v", first.Status)+ }+ if len(first.BiscuitToken) == 0 {+ t.Fatal("first biscuit token is empty")+ }++ cpPubKey := ed25519.PublicKey(first.ControlPlanePublicKey)+ trustedKeys := []ed25519.PublicKey{cpPubKey}++ // The first token must actually verify right away.+ if _, _, err := identity.VerifyBiscuitAndGetKey(first.BiscuitToken, pID, trustedKeys, srv.config.BiscuitTimeout); err != nil {+ t.Fatalf("first biscuit failed to verify immediately after issuance: %v", err)+ }++ // Let it age past BiscuitTTL, exactly as an idle router's stored token+ // would after being off for longer than the control plane's TTL. Biscuit+ // Date terms round to whole seconds, so the margin has to clear that,+ // not just the nominal 1s TTL.+ time.Sleep(3 * time.Second)++ if _, _, err := identity.VerifyBiscuitAndGetKey(first.BiscuitToken, pID, trustedKeys, srv.config.BiscuitTimeout); err == nil {+ t.Fatal("first biscuit still verifies after its TTL elapsed; test setup is not exercising an expired token")+ }++ // This is the re-enrollment call a router's reEnroll() makes after a 401+ // on lease renewal: same peer ID, same bootstrap token, fresh PoP+ // signature. It must hit the "existing enrollment request" branch (not+ // mint a brand-new one from scratch) to actually exercise the bug.+ second := doEnroll()+ if second.Status != api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {+ t.Fatalf("expected APPROVED on re-enroll, got %v", second.Status)+ }+ if bytes.Equal(second.BiscuitToken, first.BiscuitToken) {+ t.Fatal("re-enroll after expiry returned the exact same (now-expired) biscuit token instead of minting a fresh one")+ }+ if _, _, err := identity.VerifyBiscuitAndGetKey(second.BiscuitToken, pID, trustedKeys, srv.config.BiscuitTimeout); err != nil {+ t.Fatalf("refreshed biscuit from re-enroll does not verify: %v", err)+ }++ // /enroll/status must agree: it must not resurrect the stale token either.+ statusResp, err := client.Do(signedEnrollStatusRequest(t, baseURL, priv, pID.String(), time.Now().UnixMilli()))+ if err != nil {+ t.Fatal(err)+ }+ statusBody, _ := io.ReadAll(statusResp.Body)+ _ = statusResp.Body.Close()+ var polled api.BootstrapEnrollResponse+ if err := proto.Unmarshal(statusBody, &polled); err != nil {+ t.Fatalf("unmarshal polled BootstrapEnrollResponse: %v", err)+ }+ if _, _, err := identity.VerifyBiscuitAndGetKey(polled.BiscuitToken, pID, trustedKeys, srv.config.BiscuitTimeout); err != nil {+ t.Fatalf("biscuit returned by /enroll/status does not verify: %v", err)+ }++ // The enrolled node record must stay in lockstep with the refreshed+ // token: /refresh's reuse-detection compares a presented biscuit against+ // exactly this field, so a drift here would silently break that path.+ nodeRecord, err := store.GetNode(t.Context(), pID.String())+ if err != nil {+ t.Fatal(err)+ }+ if !bytes.Equal(nodeRecord.Biscuit, second.BiscuitToken) {+ t.Fatalf("enrolled node record's biscuit is out of sync with the refreshed enrollment token")+ }+}