Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane: opt-in autonomous recovery after signing-key retirement
Antonio Ojea90909c01b145
diff --git a/api/sam.pb.go b/api/sam.pb.goindex 0845d1d7..ef5db6e7 100644--- a/api/sam.pb.go+++ b/api/sam.pb.go@@ -1726,7 +1726,15 @@ type TokenRefreshRequest struct {// verifies nowhere else.ChallengeSignature []byte `protobuf:"bytes,1,opt,name=challenge_signature,json=challengeSignature,proto3" json:"challenge_signature,omitempty"`// Unix milliseconds. Must be within the control plane's freshness window.- Timestamp int64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`+ Timestamp int64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"`+ // The caller's peer ID. Optional: the control plane normally reads it+ // from the verified biscuit. It is consulted only when the biscuit's+ // signing key has been retired, so the biscuit cannot be verified: the+ // control plane then looks up the enrolled node record by this ID and,+ // if the node was opted in to autonomous recovery, accepts the request+ // when the presented biscuit is byte-identical to the last one it+ // issued and the challenge verifies against the stored public key.+ PeerId string `protobuf:"bytes,3,opt,name=peer_id,json=peerId,proto3" json:"peer_id,omitempty"`unknownFields protoimpl.UnknownFieldssizeCache protoimpl.SizeCache}@@ -1775,6 +1783,13 @@ func (x *TokenRefreshRequest) GetTimestamp() int64 {return 0}+func (x *TokenRefreshRequest) GetPeerId() string {+ if x != nil {+ return x.PeerId+ }+ return ""+}+type TokenRefreshResponse struct {state protoimpl.MessageState `protogen:"open.v1"`BiscuitToken []byte `protobuf:"bytes,1,opt,name=biscuit_token,json=biscuitToken,proto3" json:"biscuit_token,omitempty"`@@ -3081,10 +3096,11 @@ const file_api_sam_proto_rawDesc = "" +"\x05error\x18\x02 \x01(\tR\x05error\"/\n" +"\fKeysResponse\x12\x1f\n" +"\vpublic_keys\x18\x01 \x03(\fR\n" +- "publicKeys\"d\n" ++ "publicKeys\"}\n" +"\x13TokenRefreshRequest\x12/\n" +"\x13challenge_signature\x18\x01 \x01(\fR\x12challengeSignature\x12\x1c\n" +- "\ttimestamp\x18\x02 \x01(\x03R\ttimestamp\"\x7f\n" ++ "\ttimestamp\x18\x02 \x01(\x03R\ttimestamp\x12\x17\n" ++ "\apeer_id\x18\x03 \x01(\tR\x06peerId\"\x7f\n" +"\x14TokenRefreshResponse\x12#\n" +"\rbiscuit_token\x18\x01 \x01(\fR\fbiscuitToken\x12\x1d\n" +"\n" +diff --git a/api/sam.proto b/api/sam.protoindex 13d761d0..af68cd2a 100644--- a/api/sam.proto+++ b/api/sam.proto@@ -244,6 +244,14 @@ message TokenRefreshRequest {bytes challenge_signature = 1;// Unix milliseconds. Must be within the control plane's freshness window.int64 timestamp = 2;+ // The caller's peer ID. Optional: the control plane normally reads it+ // from the verified biscuit. It is consulted only when the biscuit's+ // signing key has been retired, so the biscuit cannot be verified: the+ // control plane then looks up the enrolled node record by this ID and,+ // if the node was opted in to autonomous recovery, accepts the request+ // when the presented biscuit is byte-identical to the last one it+ // issued and the challenge verifies against the stored public key.+ string peer_id = 3;}message TokenRefreshResponse {diff --git a/internal/console/public/app.js b/internal/console/public/app.jsindex 0df0ca78..2ea3b4ab 100644--- a/internal/console/public/app.js+++ b/internal/console/public/app.js@@ -243,7 +243,7 @@ async function loadData() {setTableMessage('table-users', 4, 'Restricted to administrators.');setTableMessage('table-enrollments', 4, 'Restricted to administrators.');}- renderNodesTable(data.enrolled_nodes || []);+ renderNodesTable(data.enrolled_nodes || [], role);renderServicesTable(data.node_catalog || {}, buildLabelsByPeer(data.enrolled_nodes || []));renderRoutersTable(data.active_routers || []);renderRouterTopography(data.active_routers || []);@@ -263,7 +263,7 @@ async function loadData() {console.error('Failed to load dashboard data:', error);const errMsg = `Error loading data: ${error.message}`;setTableMessage('table-users', 4, errMsg, true);- setTableMessage('table-nodes', 4, errMsg, true);+ setTableMessage('table-nodes', 5, errMsg, true);setTableMessage('table-enrollments', 4, errMsg, true);setTableMessage('table-routers', 3, errMsg, true);setTableMessage('table-bootstrap', 5, errMsg, true);@@ -366,27 +366,49 @@ function buildLabelsByPeer(nodes) {return byPeer;}-function renderNodesTable(nodes) {+function renderNodesTable(nodes, role) {const tbody = document.getElementById('table-nodes');if (nodes.length === 0) {- tbody.innerHTML = `<tr><td colspan="4" class="text-center">No enrolled nodes found</td></tr>`;+ tbody.innerHTML = `<tr><td colspan="5" class="text-center">No enrolled nodes found</td></tr>`;return;}- tbody.innerHTML = nodes.map(node => `+ tbody.innerHTML = nodes.map(node => {+ const recovery = !!node.AutonomousRecovery;+ const recoveryBadge = recovery+ ? `<span class="badge badge-approved" title="May refresh its credential after the signing key that issued it is retired">Autonomous</span>`+ : `<span class="badge badge-pending" title="Needs a new bootstrap token if offline past the key grace period">Manual</span>`;+ // The flag decides whether a lost device can rejoin on its own key, so+ // only admins get the toggle; owners just see the state.+ const toggle = role === 'admin'+ ? `<button class="btn btn-sm" onclick="setAutonomousRecovery('${escapeHTML(node.PeerID)}', ${recovery ? 'false' : 'true'})">${recovery ? 'Disable recovery' : 'Enable recovery'}</button>`+ : '';+ return `<tr><td>${peerCell(node.PeerID, node.Labels)}</td><td>${escapeHTML(node.Role)}</td><td>${escapeHTML(node.OwnerID)}</td>+ <td>${recoveryBadge}</td><td><div class="actions-cell">+ ${toggle}<button class="btn btn-sm btn-danger" onclick="revokeDevice('${escapeHTML(node.PeerID)}')">Revoke</button></div></td></tr>- `).join('');+ `;+ }).join('');}+window.setAutonomousRecovery = function(peerID, enabled) {+ const prompt = enabled+ ? 'Allow this node to refresh its credential on its own key even after the signing key that issued it is retired? A lost or stolen device with this flag can rejoin the mesh until it is revoked.'+ : 'Disable autonomous recovery? If this node is offline past the key grace period it will need a new bootstrap token to rejoin.';+ if (confirm(prompt)) {+ actionRequest(`api/admin/nodes/${peerID}/autonomous-recovery`, 'POST', { enabled }).catch(() => {});+ }+};+// node_catalog is {peerID: {services: [{name, type, description}], reported_at}},// already restricted server-side to nodes that are still admitted; type is// the short name ("mcp", "inference", "a2a") rendered by the control plane.@@ -574,6 +596,14 @@ function updateUIForRole(role, userId) {}}}+ // Same for autonomous recovery: the server refuses it from non-admins.+ const recoveryGroup = document.getElementById('token-recovery-group');+ if (recoveryGroup) {+ recoveryGroup.style.display = role === 'admin' ? 'flex' : 'none';+ if (role !== 'admin') {+ document.getElementById('token-recovery').checked = false;+ }+ }// 4. Hide users/pending sections from stats grid for normal usersconst usersCard = document.getElementById('stat-users').closest('.stat-card');@@ -699,7 +729,7 @@ function renderRouterTopography(routers) {function renderBootstrapTokensTable(tokens) {const tbody = document.getElementById('table-bootstrap');if (tokens.length === 0) {- tbody.innerHTML = `<tr><td colspan="7" class="text-center">No active bootstrap tokens found</td></tr>`;+ tbody.innerHTML = `<tr><td colspan="8" class="text-center">No active bootstrap tokens found</td></tr>`;return;}@@ -709,6 +739,7 @@ function renderBootstrapTokensTable(tokens) {const status = revoked? `<span class="badge badge-rejected">Revoked</span>`: `<span class="badge badge-approved">Active</span>`;+ const recovery = token.AutonomousRecovery ? 'Autonomous' : 'Manual';const action = revoked? '-': `<button class="btn btn-sm btn-danger" onclick="revokeBootstrapToken('${escapeHTML(token.ID)}')">Revoke</button>`;@@ -719,6 +750,7 @@ function renderBootstrapTokensTable(tokens) {<td><code>${escapeHTML(token.OwnerID || '-')}</code></td><td>${escapeHTML(String(token.UsagesCount))} / ${escapeHTML(String(token.MaxUsages))}</td><td>${escapeHTML(expiresAt)}</td>+ <td>${recovery}</td><td>${status}</td><td>${action}</td></tr>@@ -738,6 +770,7 @@ window.generateBootstrapToken = async function() {const max_usages = parseInt(document.getElementById('token-usages').value, 10);const ttl_hours = parseInt(document.getElementById('token-ttl').value, 10) || 24;const description = document.getElementById('token-desc').value;+ const autonomous_recovery = document.getElementById('token-recovery').checked;const payload = {role,@@ -749,6 +782,9 @@ window.generateBootstrapToken = async function() {if (owner_id) {payload.owner_id = owner_id;}+ if (autonomous_recovery) {+ payload.autonomous_recovery = true;+ }try {const res = await actionRequest('api/user/bootstrap-tokens', 'POST', payload);diff --git a/internal/console/public/index.html b/internal/console/public/index.htmlindex 708e50d5..5a70c540 100644--- a/internal/console/public/index.html+++ b/internal/console/public/index.html@@ -192,11 +192,12 @@<th>Node ID</th><th>Type</th><th>Owner</th>+ <th>Recovery</th><th>Actions</th></tr></thead><tbody id="table-nodes">- <tr><td colspan="4" class="text-center">Loading...</td></tr>+ <tr><td colspan="5" class="text-center">Loading...</td></tr></tbody></table></div>@@ -309,6 +310,12 @@<label for="token-desc" class="field-label">Description</label><input type="text" id="token-desc" class="field-input" placeholder="Token purpose"></div>+ <div class="input-group field" id="token-recovery-group">+ <label for="token-recovery" class="field-label">+ <input type="checkbox" id="token-recovery"> Autonomous recovery+ </label>+ <small class="field-hint">Admins only: nodes enrolled with this token may refresh their credential on their own key even after the signing key that issued it has been retired. Off by default so a node offline past the key grace period needs a new token to rejoin.</small>+ </div><button type="submit" class="btn btn-primary form-submit">Generate Token</button></form><div id="token-result" class="token-result" hidden>@@ -335,12 +342,13 @@<th>Owner</th><th>Usages</th><th>Expires</th>+ <th>Recovery</th><th>Status</th><th>Actions</th></tr></thead><tbody id="table-bootstrap">- <tr><td colspan="7" class="text-center">Loading...</td></tr>+ <tr><td colspan="8" class="text-center">Loading...</td></tr></tbody></table></div>diff --git a/internal/controlplane/biscuit_ttl_test.go b/internal/controlplane/biscuit_ttl_test.goindex c4bad393..e846e74c 100644--- a/internal/controlplane/biscuit_ttl_test.go+++ b/internal/controlplane/biscuit_ttl_test.go@@ -118,6 +118,34 @@ func registerNode(t *testing.T, cpURL, jwtToken string) (crypto.PrivKey, peer.IDfunc refreshNode(t *testing.T, cpURL string, priv crypto.PrivKey, currentBiscuit []byte) *api.TokenRefreshResponse {t.Helper()+ resp := postRefresh(t, cpURL, priv, currentBiscuit, "")+ defer func() { _ = resp.Body.Close() }()+ body, _ := io.ReadAll(resp.Body)+ if resp.StatusCode != http.StatusOK {+ t.Fatalf("/refresh status %s: %s", resp.Status, string(body))+ }++ var refreshResp api.TokenRefreshResponse+ if err := proto.Unmarshal(body, &refreshResp); err != nil {+ t.Fatalf("unmarshal TokenRefreshResponse: %v", err)+ }+ return &refreshResp+}++// refreshStatus drives /refresh like refreshNode but returns the HTTP status+// instead of failing on a non-200, for tests that expect a refusal.+func refreshStatus(t *testing.T, cpURL string, priv crypto.PrivKey, currentBiscuit []byte) int {+ t.Helper()+ resp := postRefresh(t, cpURL, priv, currentBiscuit, "")+ _ = resp.Body.Close()+ return resp.StatusCode+}++// postRefresh posts a signed TokenRefreshRequest for priv's peer. peerID, when+// non-empty, is sent in the request body's peer_id field.+func postRefresh(t *testing.T, cpURL string, priv crypto.PrivKey, currentBiscuit []byte, peerID string) *http.Response {+ t.Helper()+timestamp := time.Now().UnixMilli()pid, err := peer.IDFromPrivateKey(priv)if err != nil {@@ -130,6 +158,7 @@ func refreshNode(t *testing.T, cpURL string, priv crypto.PrivKey, currentBiscuitreqData, err := proto.Marshal(&api.TokenRefreshRequest{Timestamp: timestamp,ChallengeSignature: sig,+ PeerId: peerID,})if err != nil {t.Fatal(err)@@ -146,17 +175,7 @@ func refreshNode(t *testing.T, cpURL string, priv crypto.PrivKey, currentBiscuitif err != nil {t.Fatalf("/refresh failed: %v", err)}- defer func() { _ = resp.Body.Close() }()- body, _ := io.ReadAll(resp.Body)- if resp.StatusCode != http.StatusOK {- t.Fatalf("/refresh status %s: %s", resp.Status, string(body))- }-- var refreshResp api.TokenRefreshResponse- if err := proto.Unmarshal(body, &refreshResp); err != nil {- t.Fatalf("unmarshal TokenRefreshResponse: %v", err)- }- return &refreshResp+ return resp}// assertNear fails unless got is within a second of want, absorbing thediff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex 957dc32e..bf1c737e 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -61,6 +61,12 @@ const (// maxRequestBodyBytes caps request bodies read into memory to guard// against memory-exhaustion from oversized payloads.maxRequestBodyBytes = 1 << 20 // 1 MiB++ // adminNodeActionAutonomousRecovery is the action segment of+ // POST /admin/nodes/{peer_id}/autonomous-recovery, the per-node toggle+ // for storage.EnrolledNode.AutonomousRecovery. Admin-console only: the+ // node-facing side of #367 is TokenRefreshRequest.peer_id in sam.proto.+ adminNodeActionAutonomousRecovery = "autonomous-recovery")// Server implements the SAM Control Plane web app.@@ -217,6 +223,7 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) {mux.HandleFunc("/admin/bootstrap-tokens/", s.HandleAdminBootstrapTokenAction)mux.HandleFunc("/admin/enrollments", s.HandleAdminEnrollments)mux.HandleFunc("/admin/enrollments/", s.HandleAdminEnrollmentAction)+ mux.HandleFunc("/admin/nodes/", s.HandleAdminNodeAction)mux.HandleFunc("/admin/revoke", s.HandleAdminRevoke)mux.HandleFunc("/admin/status", s.HandleAdminStatus)mux.HandleFunc("/user/status", s.HandleUserStatus)@@ -707,19 +714,52 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {// Verify current biscuit signature and extract peer ID. Expiry is not// enforced here: a node refreshes because its token lapsed. The session// record and the signed challenge below are what bound this request.- pID, err := identity.VerifyExpiredAndExtractPeerID(trustedKeys, currentBiscuitBytes, s.config.BiscuitTimeout)- if err != nil {- logger.Warnw("Invalid biscuit presented for refresh", "error", err)- http.Error(w, "Invalid biscuit: "+err.Error(), http.StatusUnauthorized)- return+ pID, verifyErr := identity.VerifyExpiredAndExtractPeerID(trustedKeys, currentBiscuitBytes, s.config.BiscuitTimeout)+ // recovering marks the retired-key fallback (#367): the biscuit cannot be+ // verified, most likely because its signing key rotated out past grace+ // while the node was offline. The peer is then taken from the body and+ // the biscuit is authenticated below by byte-matching the last one this+ // control plane issued to that peer, which only the control plane and the+ // peer ever held.+ recovering := false+ if verifyErr != nil {+ if req.PeerId == "" {+ logger.Warnw("Invalid biscuit presented for refresh", "error", verifyErr)+ http.Error(w, "Invalid biscuit: "+verifyErr.Error(), http.StatusUnauthorized)+ return+ }+ claimed, err := peer.Decode(req.PeerId)+ if err != nil {+ http.Error(w, "Invalid Peer ID", http.StatusBadRequest)+ return+ }+ pID = claimed+ recovering = true+ } else if req.PeerId != "" {+ if claimed, err := peer.Decode(req.PeerId); err != nil || claimed != pID {+ logger.Warnw("Refresh peer_id does not match the presented biscuit", "peer_id", req.PeerId)+ http.Error(w, "peer_id does not match the presented biscuit", http.StatusUnauthorized)+ return+ }}canonical := pID.String()+ // Until the byte-match and challenge below succeed, a recovering caller+ // has proven nothing, so every refusal short of the ban check must read+ // exactly like the plain bad-biscuit one above: the fallback must not+ // become an oracle for which peer IDs are enrolled.+ unauthorized := func(msg string) {+ if recovering {+ msg = "Invalid biscuit: " + verifyErr.Error()+ }+ http.Error(w, msg, http.StatusUnauthorized)+ }+// Fetch node recordnodeRecord, err := s.store.GetNode(ctx, canonical)if err == storage.ErrNotFound {logger.Warnw("Node not found for refresh", "peer_id", canonical)- http.Error(w, "Node not enrolled", http.StatusUnauthorized)+ unauthorized("Node not enrolled")return} else if err != nil {logger.Errorf("Failed to retrieve node record: %v", err)@@ -734,7 +774,7 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {return}logger.Warnw("Session expired for node", "peer_id", canonical, "expires_at", nodeRecord.ExpiresAt)- http.Error(w, "Session expired, please re-enroll interactively", http.StatusUnauthorized)+ unauthorized("Session expired, please re-enroll interactively")return}@@ -748,20 +788,37 @@ func (s *Server) HandleRefresh(w http.ResponseWriter, r *http.Request) {if err := verifyFreshChallenge(pubKey, api.RefreshChallenge(canonical, req.Timestamp), req.Timestamp, req.ChallengeSignature); err != nil {logger.Warnw("Refresh challenge verification failed", "peer_id", canonical, "error", err)- http.Error(w, "Challenge verification failed: "+err.Error(), http.StatusUnauthorized)+ unauthorized("Challenge verification failed: " + err.Error())return}// Rotation with reuse detection: every issuance path persists the latest// biscuit, so only that one token is redeemable. A replayed refresh// presents an already-rotated biscuit and is refused; a node that lost the- // rotated token recovers through its re-enrollment fallback.+ // rotated token recovers through its re-enrollment fallback. In the+ // retired-key fallback this is also what authenticates the biscuit at+ // all, standing in for the signature that could not be checked.if subtle.ConstantTimeCompare(currentBiscuitBytes, nodeRecord.Biscuit) != 1 {logger.Warnw("Refresh presented an already-rotated biscuit (possible replay)", "peer_id", nodeRecord.PeerID)- http.Error(w, "Biscuit already rotated: only the latest issued token can be refreshed, re-enroll instead", http.StatusUnauthorized)+ unauthorized("Biscuit already rotated: only the latest issued token can be refreshed, re-enroll instead")return}+ // The caller has now proven it holds both the node key and the last+ // issued biscuit. Whether that is enough to survive the retired signing+ // key is the operator's call: off by default, because a node that can+ // always come back on its own private key holds a credential that never+ // expires, and the key grace period is what otherwise puts a deadline on+ // a stolen or forgotten machine.+ if recovering {+ if !nodeRecord.AutonomousRecovery {+ logger.Warnw("Refused retired-key refresh: node is not opted in to autonomous recovery", "peer_id", canonical, "error", verifyErr)+ http.Error(w, "Biscuit signing key has been retired and this node is not opted in to autonomous recovery: re-enroll with a new bootstrap token", http.StatusUnauthorized)+ return+ }+ logger.Infow("Autonomous recovery: re-issuing a biscuit whose signing key was retired", "peer_id", canonical, "error", verifyErr)+ }+// Fetch current signing private key and policy configprivKey, _, err := s.store.GetCurrentKey(ctx)if err != nil {@@ -1400,15 +1457,16 @@ func (s *Server) HandleEnroll(w http.ResponseWriter, r *http.Request) {}nodeRecord := &storage.EnrolledNode{- PeerID: canonical,- PublicKey: req.PublicKey,- Biscuit: biscuitBytes,- Role: tokenRecord.Role,- OwnerID: tokenRecord.OwnerID,- EnrollmentType: "BOOTSTRAP",- Labels: req.Labels,- EnrolledAt: time.Now(),- ExpiresAt: time.Time{},+ PeerID: canonical,+ PublicKey: req.PublicKey,+ Biscuit: biscuitBytes,+ Role: tokenRecord.Role,+ OwnerID: tokenRecord.OwnerID,+ EnrollmentType: "BOOTSTRAP",+ Labels: req.Labels,+ EnrolledAt: time.Now(),+ ExpiresAt: time.Time{},+ AutonomousRecovery: tokenRecord.AutonomousRecovery,}if err := s.store.EnrollNode(ctx, nodeRecord); err != nil {logger.Errorf("Failed to enroll active bootstrap node: %v", err)@@ -1660,6 +1718,9 @@ func (s *Server) HandleAdminBootstrapTokens(w http.ResponseWriter, r *http.RequeTTLHours int `json:"ttl_hours"`MaxUsages int `json:"max_usages"`Description string `json:"description"`+ // Copied onto every node this token enrolls; see+ // storage.EnrolledNode.AutonomousRecovery.+ AutonomousRecovery bool `json:"autonomous_recovery"`}r.Body = http.MaxBytesReader(w, r.Body, maxRequestBodyBytes)@@ -1688,14 +1749,15 @@ func (s *Server) HandleAdminBootstrapTokens(w http.ResponseWriter, r *http.RequetokenID := fmt.Sprintf("%x", sha256.Sum256([]byte(tokenVal)))tokenRecord := &storage.BootstrapToken{- ID: tokenID,- TokenHash: tokenID,- Role: req.Role,- MaxUsages: req.MaxUsages,- UsagesCount: 0,- Description: req.Description,- CreatedAt: time.Now(),- ExpiresAt: time.Now().Add(time.Duration(req.TTLHours) * time.Hour),+ ID: tokenID,+ TokenHash: tokenID,+ Role: req.Role,+ MaxUsages: req.MaxUsages,+ UsagesCount: 0,+ Description: req.Description,+ CreatedAt: time.Now(),+ ExpiresAt: time.Now().Add(time.Duration(req.TTLHours) * time.Hour),+ AutonomousRecovery: req.AutonomousRecovery,}if err := s.store.SaveBootstrapToken(r.Context(), tokenRecord); err != nil {@@ -1880,15 +1942,16 @@ func (s *Server) HandleAdminEnrollmentAction(w http.ResponseWriter, r *http.Requ}nodeRecord := &storage.EnrolledNode{- PeerID: canonical,- PublicKey: enrollReq.PublicKey,- Biscuit: biscuitBytes,- Role: tokenRecord.Role,- OwnerID: tokenRecord.OwnerID,- EnrollmentType: "BOOTSTRAP",- Labels: enrollReq.Labels,- EnrolledAt: time.Now(),- ExpiresAt: time.Time{},+ PeerID: canonical,+ PublicKey: enrollReq.PublicKey,+ Biscuit: biscuitBytes,+ Role: tokenRecord.Role,+ OwnerID: tokenRecord.OwnerID,+ EnrollmentType: "BOOTSTRAP",+ Labels: enrollReq.Labels,+ EnrolledAt: time.Now(),+ ExpiresAt: time.Time{},+ AutonomousRecovery: tokenRecord.AutonomousRecovery,}if err := s.store.EnrollNode(ctx, nodeRecord); err != nil {logger.Errorf("Failed to enroll active bootstrap node: %v", err)@@ -1908,6 +1971,55 @@ func (s *Server) HandleAdminEnrollmentAction(w http.ResponseWriter, r *http.Requhttp.Error(w, "Invalid action", http.StatusBadRequest)}+// HandleAdminNodeAction HTTP POST `/admin/nodes/{peer_id}/autonomous-recovery`+// with body {"enabled": bool} toggles storage.EnrolledNode.AutonomousRecovery+// for one enrolled node. This is the per-node counterpart of the flag on a+// bootstrap token, for a node that is already enrolled.+func (s *Server) HandleAdminNodeAction(w http.ResponseWriter, r *http.Request) {+ if !s.checkAdminAuth(w, r) {+ return+ }++ if r.Method != http.MethodPost {+ http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)+ return+ }++ parts := strings.Split(strings.TrimPrefix(r.URL.Path, "/admin/nodes/"), "/")+ if len(parts) != 2 || parts[1] != adminNodeActionAutonomousRecovery {+ http.Error(w, "Not found", http.StatusNotFound)+ return+ }+ pID, err := peer.Decode(parts[0])+ if err != nil {+ http.Error(w, "Invalid Peer ID", http.StatusBadRequest)+ return+ }++ var req struct {+ Enabled bool `json:"enabled"`+ }+ r.Body = http.MaxBytesReader(w, r.Body, maxRequestBodyBytes)+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ http.Error(w, "Invalid JSON body", http.StatusBadRequest)+ return+ }+ defer func() { _ = r.Body.Close() }()++ canonical := pID.String()+ err = s.store.SetNodeAutonomousRecovery(r.Context(), canonical, req.Enabled)+ if err == storage.ErrNotFound {+ http.Error(w, "Node not found", http.StatusNotFound)+ return+ } else if err != nil {+ logger.Errorf("Failed to set autonomous recovery for node %s: %v", canonical, err)+ http.Error(w, "Internal server error", http.StatusInternalServerError)+ return+ }+ logger.Infow("Autonomous recovery toggled", "peer_id", canonical, "enabled", req.Enabled)+ w.WriteHeader(http.StatusNoContent)+}+// HandleAdminRevoke HTTP POST `/admin/revoke`func (s *Server) HandleAdminRevoke(w http.ResponseWriter, r *http.Request) {if !s.checkAdminAuth(w, r) {@@ -2197,6 +2309,10 @@ func (s *Server) HandleUserBootstrapTokens(w http.ResponseWriter, r *http.RequesTTLHours int `json:"ttl_hours"`MaxUsages int `json:"max_usages"`Description string `json:"description"`+ // Copied onto every node this token enrolls; see+ // storage.EnrolledNode.AutonomousRecovery. Admin-only: it decides+ // whether a lost device can rejoin the mesh on its own.+ AutonomousRecovery bool `json:"autonomous_recovery"`}r.Body = http.MaxBytesReader(w, r.Body, maxRequestBodyBytes)@@ -2214,6 +2330,10 @@ func (s *Server) HandleUserBootstrapTokens(w http.ResponseWriter, r *http.Requeshttp.Error(w, "Forbidden: Standard users can only generate tokens for node or box roles", http.StatusForbidden)return}+ if user.Role != "admin" && req.AutonomousRecovery {+ http.Error(w, "Forbidden: only admins can issue tokens with autonomous_recovery", http.StatusForbidden)+ return+ }ownerID, status, err := s.resolveTokenOwner(r.Context(), user, req.OwnerID)if err != nil {@@ -2237,15 +2357,16 @@ func (s *Server) HandleUserBootstrapTokens(w http.ResponseWriter, r *http.RequestokenID := fmt.Sprintf("%x", sha256.Sum256([]byte(tokenVal)))tokenRecord := &storage.BootstrapToken{- ID: tokenID,- TokenHash: tokenID,- Role: req.Role,- OwnerID: ownerID,- MaxUsages: req.MaxUsages,- UsagesCount: 0,- Description: req.Description,- CreatedAt: time.Now(),- ExpiresAt: time.Now().Add(time.Duration(req.TTLHours) * time.Hour),+ ID: tokenID,+ TokenHash: tokenID,+ Role: req.Role,+ OwnerID: ownerID,+ MaxUsages: req.MaxUsages,+ UsagesCount: 0,+ Description: req.Description,+ CreatedAt: time.Now(),+ ExpiresAt: time.Now().Add(time.Duration(req.TTLHours) * time.Hour),+ AutonomousRecovery: req.AutonomousRecovery,}if err := s.store.SaveBootstrapToken(r.Context(), tokenRecord); err != nil {diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.goindex dd3465f0..9c7d42d6 100644--- a/internal/controlplane/server_test.go+++ b/internal/controlplane/server_test.go@@ -2784,6 +2784,13 @@ func TestHandleInfoPublishesBanSet(t *testing.T) {func createAdminBootstrapToken(t *testing.T, baseURL, adminToken, role string, maxUsages int) string {t.Helper()body := fmt.Sprintf(`{"role":%q,"ttl_hours":2,"max_usages":%d,"description":"re-enroll test"}`, role, maxUsages)+ return createAdminBootstrapTokenJSON(t, baseURL, adminToken, body)+}++// createAdminBootstrapTokenJSON posts body verbatim to /admin/bootstrap-tokens+// and returns the plaintext token string.+func createAdminBootstrapTokenJSON(t *testing.T, baseURL, adminToken, body string) string {+ t.Helper()req, err := http.NewRequest(http.MethodPost, baseURL+"/admin/bootstrap-tokens", bytes.NewBufferString(body))if err != nil {t.Fatal(err)@@ -2975,6 +2982,48 @@ func TestReEnrollAfterExpiryMintsFreshBiscuit(t *testing.T) {}})+ t.Run("re-enroll recovers a node whose signing key retired past grace", func(t *testing.T) {+ // The #367 scenario: the node's biscuit is signed by a key the+ // control plane has since retired, so /refresh structurally cannot+ // accept it. The manual recovery lever is a fresh bootstrap token,+ // and re-enroll must then mint under the *active* key.+ srv, store, baseURL, token := start(t, api.RoleRouter, 5)+ priv, pID, pubBytes := newPeer(t)++ first := bootstrapEnroll(t, baseURL, token, priv, pID, pubBytes, api.RoleRouter, nil)+ oldKey := ed25519.PublicKey(first.ControlPlanePublicKey)++ newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ // A negative grace period retires the old key immediately.+ if err := store.RotateKeys(context.Background(), newPriv, newPub, -time.Second); err != nil {+ t.Fatal(err)+ }++ if code := refreshStatus(t, baseURL, priv, first.BiscuitToken); code != http.StatusUnauthorized {+ t.Fatalf("/refresh with a retired-key biscuit: got %d, want 401", code)+ }++ second := bootstrapEnroll(t, baseURL, token, priv, pID, pubBytes, api.RoleRouter, nil)+ if second.Status != api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {+ t.Fatalf("re-enroll after key retirement: got %v, want APPROVED", second.Status)+ }+ if !bytes.Equal(second.ControlPlanePublicKey, newPub) {+ t.Fatal("re-enroll response did not advertise the active signing key")+ }+ if _, _, err := identity.VerifyBiscuitAndGetKey(second.BiscuitToken, pID, []ed25519.PublicKey{oldKey}, srv.config.BiscuitTimeout); err == nil {+ t.Fatal("re-minted biscuit still verifies under the retired key")+ }+ if _, _, err := identity.VerifyBiscuitAndGetKey(second.BiscuitToken, pID, []ed25519.PublicKey{newPub}, srv.config.BiscuitTimeout); err != nil {+ t.Fatalf("re-minted biscuit does not verify under the active key: %v", err)+ }+ if refreshed := refreshNode(t, baseURL, priv, second.BiscuitToken); len(refreshed.BiscuitToken) == 0 {+ t.Fatal("refresh after key-retirement re-enroll returned an empty biscuit")+ }+ })+t.Run("re-enroll ignores the request's own declared labels", func(t *testing.T) {_, store, baseURL, token := start(t, api.RoleRouter, 5)priv, pID, pubBytes := newPeer(t)@@ -3148,3 +3197,340 @@ func TestAdminRevokeBootstrapToken(t *testing.T) {t.Errorf("re-enroll with a revoked token: got %v, want REJECTED", out.Status)}}++// TestAutonomousRecovery pins the opt-in retired-key fallback on /refresh+// (#367). A bootstrap node whose biscuit is signed by a key the control plane+// has since retired cannot have that biscuit verified, so /refresh normally+// refuses it and the node needs an operator to mint a new bootstrap token.+// With autonomous_recovery set on the node record - seeded from the token+// that enrolled it, or toggled per node by an admin - the control plane+// instead identifies the node by TokenRefreshRequest.peer_id, authenticates+// the biscuit by byte-matching the last one it issued, verifies the fresh+// challenge against the stored key, re-checks admission, and mints under the+// active key. Off by default: a node that can always come back on its own+// key holds a credential that never expires.+func TestAutonomousRecovery(t *testing.T) {+ issuer, mintToken := startCustomMockOIDC(t)+ const adminToken = "super-secret-admin-token"++ type fixture struct {+ srv *Server+ store storage.Store+ baseURL string+ priv crypto.PrivKey+ pID peer.ID+ biscuit []byte+ cpKey ed25519.PublicKey+ }++ // enroll brings up a fresh control plane, enrolls a router with a token+ // minted with the given autonomous_recovery flag, and returns everything+ // a subtest needs to drive /refresh for it.+ enroll := func(t *testing.T, autonomous bool) *fixture {+ t.Helper()+ srv, store, baseURL := setupTestServer(t, issuer)+ t.Cleanup(func() {+ _ = srv.Close()+ _ = store.Close()+ })+ srv.config.AdminToken = adminToken+ srv.config.AutoApproveEnrollment = true++ body := fmt.Sprintf(`{"role":%q,"ttl_hours":2,"max_usages":5,"autonomous_recovery":%t}`, api.RoleRouter, autonomous)+ token := createAdminBootstrapTokenJSON(t, baseURL, adminToken, body)++ priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ if err != nil {+ t.Fatal(err)+ }+ pID, err := peer.IDFromPrivateKey(priv)+ if err != nil {+ t.Fatal(err)+ }+ pubBytes, err := crypto.MarshalPublicKey(pub)+ if err != nil {+ t.Fatal(err)+ }+ resp := bootstrapEnroll(t, baseURL, token, priv, pID, pubBytes, api.RoleRouter, nil)+ if resp.Status != api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {+ t.Fatalf("enroll: got %v, want APPROVED", resp.Status)+ }++ record, err := store.GetNode(context.Background(), pID.String())+ if err != nil {+ t.Fatal(err)+ }+ if record.AutonomousRecovery != autonomous {+ t.Fatalf("node record AutonomousRecovery = %t, want %t (must be copied from the token)", record.AutonomousRecovery, autonomous)+ }+ return &fixture{srv: srv, store: store, baseURL: baseURL, priv: priv, pID: pID, biscuit: resp.BiscuitToken, cpKey: ed25519.PublicKey(resp.ControlPlanePublicKey)}+ }++ // retireSigningKey rotates to a new key with a negative grace period so+ // the key that signed every biscuit issued so far is gone at once.+ retireSigningKey := func(t *testing.T, store storage.Store) ed25519.PublicKey {+ t.Helper()+ newPub, newPriv, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ if err := store.RotateKeys(context.Background(), newPriv, newPub, -time.Second); err != nil {+ t.Fatal(err)+ }+ return newPub+ }++ // refresh drives /refresh with peer_id set, the way a node does, and+ // returns status and body.+ refresh := func(t *testing.T, f *fixture, biscuit []byte) (int, []byte) {+ t.Helper()+ resp := postRefresh(t, f.baseURL, f.priv, biscuit, f.pID.String())+ defer func() { _ = resp.Body.Close() }()+ body, _ := io.ReadAll(resp.Body)+ return resp.StatusCode, body+ }++ setRecovery := func(t *testing.T, f *fixture, peerID string, enabled bool) int {+ t.Helper()+ req, err := http.NewRequest(http.MethodPost, f.baseURL+"/admin/nodes/"+peerID+"/autonomous-recovery", bytes.NewBufferString(fmt.Sprintf(`{"enabled":%t}`, enabled)))+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", "Bearer "+adminToken)+ resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req)+ if err != nil {+ t.Fatal(err)+ }+ _ = resp.Body.Close()+ return resp.StatusCode+ }++ t.Run("opted in via token recovers across key retirement", func(t *testing.T) {+ f := enroll(t, true)+ newKey := retireSigningKey(t, f.store)++ code, body := refresh(t, f, f.biscuit)+ if code != http.StatusOK {+ t.Fatalf("/refresh after key retirement: got %d %s, want 200", code, body)+ }+ var out api.TokenRefreshResponse+ if err := proto.Unmarshal(body, &out); err != nil {+ t.Fatal(err)+ }+ if _, _, err := identity.VerifyBiscuitAndGetKey(out.BiscuitToken, f.pID, []ed25519.PublicKey{newKey}, f.srv.config.BiscuitTimeout); err != nil {+ t.Fatalf("recovered biscuit does not verify under the active key: %v", err)+ }+ if _, _, err := identity.VerifyBiscuitAndGetKey(out.BiscuitToken, f.pID, []ed25519.PublicKey{f.cpKey}, f.srv.config.BiscuitTimeout); err == nil {+ t.Fatal("recovered biscuit still verifies under the retired key")+ }+ // And the recovered biscuit is a normal one from here on.+ if next := refreshNode(t, f.baseURL, f.priv, out.BiscuitToken); len(next.BiscuitToken) == 0 {+ t.Fatal("refresh after recovery returned an empty biscuit")+ }+ })++ t.Run("not opted in is refused and told to re-enroll", func(t *testing.T) {+ f := enroll(t, false)+ retireSigningKey(t, f.store)++ code, body := refresh(t, f, f.biscuit)+ if code != http.StatusUnauthorized {+ t.Fatalf("/refresh without opt-in: got %d %s, want 401", code, body)+ }+ if !strings.Contains(string(body), "not opted in to autonomous recovery") {+ t.Fatalf("refusal should tell the operator what to change, got: %s", body)+ }+ record, err := f.store.GetNode(context.Background(), f.pID.String())+ if err != nil {+ t.Fatal(err)+ }+ if !bytes.Equal(record.Biscuit, f.biscuit) {+ t.Fatal("a refused recovery must not rotate the stored biscuit")+ }+ })++ t.Run("banned stays out even when opted in", func(t *testing.T) {+ f := enroll(t, true)+ retireSigningKey(t, f.store)+ if err := f.store.SetNodeBanned(context.Background(), f.pID.String(), true); err != nil {+ t.Fatal(err)+ }++ code, body := refresh(t, f, f.biscuit)+ if code != http.StatusForbidden {+ t.Fatalf("banned node's recovery: got %d %s, want 403", code, body)+ }+ })++ t.Run("a replayed superseded biscuit is refused", func(t *testing.T) {+ f := enroll(t, true)+ // Rotate B1 -> B2 while the key is still live, then retire the key.+ // Only B2 is redeemable; a captured B1 must not recover anything.+ second := refreshNode(t, f.baseURL, f.priv, f.biscuit)+ retireSigningKey(t, f.store)++ code, body := refresh(t, f, f.biscuit)+ if code != http.StatusUnauthorized {+ t.Fatalf("replayed B1: got %d %s, want 401", code, body)+ }+ if code, body := refresh(t, f, second.BiscuitToken); code != http.StatusOK {+ t.Fatalf("B2 recovery: got %d %s, want 200", code, body)+ }+ })++ t.Run("recovery needs the node key, not just the biscuit", func(t *testing.T) {+ f := enroll(t, true)+ retireSigningKey(t, f.store)++ // Same peer_id and last-issued biscuit, but the challenge is signed+ // by someone else's key.+ thief, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ if err != nil {+ t.Fatal(err)+ }+ ts := time.Now().UnixMilli()+ sig, err := thief.Sign(api.RefreshChallenge(f.pID.String(), ts))+ if err != nil {+ t.Fatal(err)+ }+ reqData, err := proto.Marshal(&api.TokenRefreshRequest{Timestamp: ts, ChallengeSignature: sig, PeerId: f.pID.String()})+ if err != nil {+ t.Fatal(err)+ }+ if code, _ := doRefresh(t, f.baseURL, f.biscuit, reqData); code != http.StatusUnauthorized {+ t.Fatalf("stolen-biscuit recovery: got %d, want 401", code)+ }+ })++ t.Run("fallback does not reveal which peers are enrolled", func(t *testing.T) {+ f := enroll(t, true)+ retireSigningKey(t, f.store)++ // Garbage bearer, no peer_id: the pre-#367 refusal.+ garbage := []byte("not-a-biscuit")+ baseline := postRefresh(t, f.baseURL, f.priv, garbage, "")+ baselineBody, _ := io.ReadAll(baseline.Body)+ _ = baseline.Body.Close()+ if baseline.StatusCode != http.StatusUnauthorized {+ t.Fatalf("baseline: got %d, want 401", baseline.StatusCode)+ }++ // Garbage bearer with an enrolled peer_id: byte-match fails.+ enrolledResp := postRefresh(t, f.baseURL, f.priv, garbage, f.pID.String())+ enrolledBody, _ := io.ReadAll(enrolledResp.Body)+ _ = enrolledResp.Body.Close()++ // Garbage bearer with a never-enrolled peer_id: record lookup fails.+ stranger, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ if err != nil {+ t.Fatal(err)+ }+ strangerID, err := peer.IDFromPrivateKey(stranger)+ if err != nil {+ t.Fatal(err)+ }+ strangerResp := postRefresh(t, f.baseURL, stranger, garbage, strangerID.String())+ strangerBody, _ := io.ReadAll(strangerResp.Body)+ _ = strangerResp.Body.Close()++ for name, got := range map[string]struct {+ code int+ body []byte+ }{+ "enrolled peer": {enrolledResp.StatusCode, enrolledBody},+ "unknown peer": {strangerResp.StatusCode, strangerBody},+ } {+ if got.code != baseline.StatusCode || !bytes.Equal(got.body, baselineBody) {+ t.Errorf("%s: got %d %q, want the baseline %d %q", name, got.code, got.body, baseline.StatusCode, baselineBody)+ }+ }+ })++ t.Run("admin toggle opts an enrolled node in and out", func(t *testing.T) {+ f := enroll(t, false)++ if code := setRecovery(t, f, f.pID.String(), true); code != http.StatusNoContent {+ t.Fatalf("enable: got %d, want 204", code)+ }+ retireSigningKey(t, f.store)+ code, body := refresh(t, f, f.biscuit)+ if code != http.StatusOK {+ t.Fatalf("/refresh after admin opt-in: got %d %s, want 200", code, body)+ }+ var out api.TokenRefreshResponse+ if err := proto.Unmarshal(body, &out); err != nil {+ t.Fatal(err)+ }++ // A refresh must not clear the flag on its way through EnrollNode.+ record, err := f.store.GetNode(context.Background(), f.pID.String())+ if err != nil {+ t.Fatal(err)+ }+ if !record.AutonomousRecovery {+ t.Fatal("refresh cleared AutonomousRecovery")+ }++ if code := setRecovery(t, f, f.pID.String(), false); code != http.StatusNoContent {+ t.Fatalf("disable: got %d, want 204", code)+ }+ retireSigningKey(t, f.store)+ if code, body := refresh(t, f, out.BiscuitToken); code != http.StatusUnauthorized {+ t.Fatalf("/refresh after admin opt-out: got %d %s, want 401", code, body)+ }++ unknown, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ if err != nil {+ t.Fatal(err)+ }+ unknownID, err := peer.IDFromPrivateKey(unknown)+ if err != nil {+ t.Fatal(err)+ }+ if code := setRecovery(t, f, unknownID.String(), true); code != http.StatusNotFound {+ t.Fatalf("toggle on a never-enrolled peer: got %d, want 404", code)+ }+ if code := setRecovery(t, f, "not-a-peer-id", true); code != http.StatusBadRequest {+ t.Fatalf("toggle on a malformed peer id: got %d, want 400", code)+ }+ })++ t.Run("peer_id must match a verifiable biscuit", func(t *testing.T) {+ f := enroll(t, true)+ other, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ if err != nil {+ t.Fatal(err)+ }+ otherID, err := peer.IDFromPrivateKey(other)+ if err != nil {+ t.Fatal(err)+ }+ resp := postRefresh(t, f.baseURL, f.priv, f.biscuit, otherID.String())+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusUnauthorized {+ t.Fatalf("valid biscuit with a foreign peer_id: got %d, want 401", resp.StatusCode)+ }+ })++ t.Run("only admins may mint tokens with autonomous_recovery", func(t *testing.T) {+ f := enroll(t, false)+ // A non-admin OIDC user hits the user endpoint.+ userJWT := mintToken(map[string]interface{}{"sub": "plain-user", "email": "[email protected]"})+ body := fmt.Sprintf(`{"role":%q,"autonomous_recovery":true}`, api.RoleNode)+ req, err := http.NewRequest(http.MethodPost, f.baseURL+"/user/bootstrap-tokens", bytes.NewBufferString(body))+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", "Bearer "+userJWT)+ resp, err := (&http.Client{Timeout: 5 * time.Second}).Do(req)+ if err != nil {+ t.Fatal(err)+ }+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusForbidden {+ t.Fatalf("non-admin minting an autonomous_recovery token: got %d, want 403", resp.StatusCode)+ }+ })+}diff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.goindex d4afd929..c63aa50a 100644--- a/internal/node/enroll_test.go+++ b/internal/node/enroll_test.go@@ -18,6 +18,7 @@ import ("bytes""context""crypto/ed25519"+ "encoding/base64""encoding/json""io""net/http"@@ -234,6 +235,145 @@ func TestStartRecoversStaleIdentityViaRefreshToken(t *testing.T) {}}+// TestStartRecoversStaleIdentityViaAutonomousRefresh covers #367 from the+// node's side: a bootstrap node has no OIDC refresh grant, so when its+// stored identity is signed by a retired key the only silent recovery is+// /refresh with peer_id set, which the control plane honours if the operator+// opted the node in. The node must send peer_id and adopt the result.+func TestStartRecoversStaleIdentityViaAutonomousRefresh(t *testing.T) {+ cpPub, cpPriv, _ := ed25519.GenerateKey(nil)+ _, stalePriv, _ := ed25519.GenerateKey(nil)++ routerAddr := startMockRouterWithKey(t, cpPriv, api.RoleRouter)++ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = store.Close() }()++ privKey := GetOrGenerateKey(store)+ wantPeerID, err := peer.IDFromPublicKey(privKey.GetPublic())+ if err != nil {+ t.Fatal(err)+ }++ mint := func(priv ed25519.PrivateKey, peerID string) []byte {+ builder := biscuit.NewBuilder(priv)+ for _, f := range []biscuit.Fact{+ {Predicate: biscuit.Predicate{Name: api.FactNode, IDs: []biscuit.Term{biscuit.String(peerID)}}},+ {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(24 * time.Hour))}}},+ } {+ if err := builder.AddAuthorityFact(f); err != nil {+ t.Fatalf("failed to add fact: %v", err)+ }+ }+ tok, err := builder.Build()+ if err != nil {+ t.Fatalf("failed to build biscuit: %v", err)+ }+ b, err := tok.Serialize()+ if err != nil {+ t.Fatalf("failed to serialize biscuit: %v", err)+ }+ return b+ }++ staleBiscuit := mint(stalePriv, wantPeerID.String())+ if err := store.SaveIdentity(staleBiscuit); err != nil {+ t.Fatal(err)+ }++ // Mock control plane: /refresh behaves like the opted-in fallback - it+ // cannot verify the bearer, finds the node by peer_id, checks the bearer+ // is the last one it issued and the challenge is signed by the node key.+ var mu sync.Mutex+ var gotPeerID string+ var gotBearer []byte+ cpMux := http.NewServeMux()+ cpMux.HandleFunc("/refresh", func(w http.ResponseWriter, r *http.Request) {+ bearer, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "))+ if err != nil {+ http.Error(w, "bad bearer", http.StatusBadRequest)+ return+ }+ body, _ := io.ReadAll(r.Body)+ var req api.TokenRefreshRequest+ if err := proto.Unmarshal(body, &req); err != nil {+ http.Error(w, "bad request", http.StatusBadRequest)+ return+ }+ mu.Lock()+ gotPeerID, gotBearer = req.PeerId, bearer+ mu.Unlock()+ if req.PeerId != wantPeerID.String() || !bytes.Equal(bearer, staleBiscuit) {+ http.Error(w, "Invalid biscuit", http.StatusUnauthorized)+ return+ }+ ok, err := privKey.GetPublic().Verify(api.RefreshChallenge(req.PeerId, req.Timestamp), req.ChallengeSignature)+ if err != nil || !ok {+ http.Error(w, "Challenge verification failed", http.StatusUnauthorized)+ return+ }+ data, _ := proto.Marshal(&api.TokenRefreshResponse{+ BiscuitToken: mint(cpPriv, req.PeerId),+ ExpiresAt: time.Now().Add(24 * time.Hour).Unix(),+ })+ w.Header().Set("Content-Type", "application/x-protobuf")+ _, _ = w.Write(data)+ })+ cpMux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {+ t.Error("/register must not be hit: a bootstrap node has no OIDC grant to re-enroll with")+ http.Error(w, "unexpected", http.StatusInternalServerError)+ })+ cpSrv := httptest.NewServer(cpMux)+ defer cpSrv.Close()++ if err := store.SaveControlPlaneURL(cpSrv.URL); err != nil {+ t.Fatal(err)+ }+ if err := store.SaveMeshConfig(cpPub, []string{routerAddr}); err != nil {+ t.Fatal(err)+ }++ node, err := NewSamNode(Options{+ PrivKey: privKey,+ Store: store,+ ControlPlanePubKey: cpPub,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ })+ if err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ if err := node.Start(ctx); err != nil {+ t.Fatalf("Start should have recovered the stale identity via /refresh, got: %v", err)+ }++ mu.Lock()+ if gotPeerID != wantPeerID.String() {+ t.Errorf("/refresh peer_id: got %q, want %q", gotPeerID, wantPeerID)+ }+ if !bytes.Equal(gotBearer, staleBiscuit) {+ t.Error("/refresh must present the stored (stale) biscuit for the control plane to byte-match")+ }+ mu.Unlock()++ if err := identity.VerifyBiscuitRole(node.GetIdentity(), cpPub, api.RoleNode, time.Second); err != nil {+ t.Errorf("recovered identity does not verify under the current CP key: %v", err)+ }+ stored, err := store.LoadIdentity()+ if err != nil {+ t.Fatal(err)+ }+ if !bytes.Equal(stored, node.GetIdentity()) {+ t.Error("in-memory identity and stored identity diverged after recovery")+ }+}+// Without a refresh token the startup failure must stay fatal.func TestStartStaleIdentityWithoutRefreshTokenFails(t *testing.T) {cpPub, _, _ := ed25519.GenerateKey(nil)diff --git a/internal/node/node.go b/internal/node/node.goindex b784588d..6ca1b088 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -354,13 +354,21 @@ func (n *SamNode) Start(ctx context.Context) error {}if roleErr != nil {// Stale identity (e.g. signing key rotated past its grace- // period while offline): try silent re-enrollment with the- // stored refresh token before giving up.- logger.Warnf("Loaded identity fails role requirement %q: %v; attempting recovery via stored refresh token", n.config.RequiredRole, roleErr)- if recErr := n.ReEnrollWithRefreshToken(ctx); recErr != nil {- return fmt.Errorf("loaded identity fails role requirement %q (refresh-token recovery failed: %v): %w", n.config.RequiredRole, recErr, roleErr)+ // period while offline). Two silent recoveries are tried+ // before giving up: /refresh, which the control plane honours+ // on proof of possession alone if an operator opted this node+ // in to autonomous recovery (the only path a bootstrap node+ // has), then re-enrollment with the stored OIDC refresh token.+ logger.Warnf("Loaded identity fails role requirement %q: %v; attempting recovery", n.config.RequiredRole, roleErr)+ if refreshErr := n.RefreshEnrollment(ctx); refreshErr == nil {+ logger.Info("Identity recovered via autonomous refresh.")+ } else {+ logger.Warnf("Autonomous refresh refused (%v); attempting recovery via stored refresh token", refreshErr)+ if recErr := n.ReEnrollWithRefreshToken(ctx); recErr != nil {+ return fmt.Errorf("loaded identity fails role requirement %q (autonomous refresh: %v; refresh-token recovery: %v): %w", n.config.RequiredRole, refreshErr, recErr, roleErr)+ }+ logger.Info("Identity recovered via refresh-token re-enrollment.")}- logger.Info("Identity recovered via refresh-token re-enrollment.")// Re-enrollment persisted the response's router addresses; adopt// them so the static relay setup below doesn't use stale ones.if _, storedAddrs, loadErr := n.Store.LoadMeshConfig(); loadErr == nil {@@ -1081,10 +1089,14 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {return fmt.Errorf("failed to generate signature: %w", err)}- // 4. Construct request+ // 4. Construct request. peer_id lets the control plane find this node's+ // record when the biscuit's signing key has been retired and the biscuit+ // itself can no longer be verified (autonomous recovery, opt-in+ // server-side); it is cross-checked against the biscuit otherwise.req := &api.TokenRefreshRequest{ChallengeSignature: sig,Timestamp: timestamp,+ PeerId: peerID.String(),}reqData, err := proto.Marshal(req)if err != nil {@@ -1151,6 +1163,7 @@ func (n *SamNode) RefreshEnrollment(ctx context.Context) error {if err := n.Store.SaveIdentity(refreshResp.BiscuitToken); err != nil {return fmt.Errorf("failed to save refreshed identity: %w", err)}+ n.SetIdentityCache(refreshResp.BiscuitToken)if err := n.Store.SaveIdentityExpiration(refreshResp.ExpiresAt); err != nil {return fmt.Errorf("failed to save refreshed expiration: %w", err)}diff --git a/internal/router/router.go b/internal/router/router.goindex bce3d493..8e7797e2 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -619,6 +619,23 @@ func (r *Router) reEnroll() error {return r.enrollWithTokens(r.Host.ID())}+// recoverAfterLease401 restores the router's standing with the control plane+// after a lease renewal was refused. A refresh goes first: it succeeds on+// proof of possession alone when the biscuit's signing key was retired and+// the operator opted this router in to autonomous recovery, and it already+// falls back to re-enrollment on a 401 of its own. Any other failure still+// ends in re-enrollment with the bootstrap token, the pre-existing contract.+func (r *Router) recoverAfterLease401() error {+ if r.privKey != nil && r.ctx != nil {+ err := r.RefreshEnrollment(r.ctx)+ if err == nil {+ return nil+ }+ logger.Warnf("Refresh after lease 401 failed (%v), re-enrolling", err)+ }+ return r.reEnroll()+}+func (r *Router) syncKeys() error {client := &http.Client{Timeout: 10 * time.Second}resp, err := client.Get(r.config.ControlPlaneURL + "/keys")@@ -827,12 +844,12 @@ func (r *Router) renewLease() {_ = resp.Body.Close()if resp.StatusCode == http.StatusUnauthorized && attempt == 0 {- logger.Warnf("Control plane lease renewal rejected (401 Unauthorized: %s), attempting re-enrollment...", string(body))- if err := r.reEnroll(); err != nil {- logger.Errorf("Re-enrollment failed after 401 Unauthorized lease renewal: %v", err)+ logger.Warnf("Control plane lease renewal rejected (401 Unauthorized: %s), attempting recovery...", string(body))+ if err := r.recoverAfterLease401(); err != nil {+ logger.Errorf("Recovery failed after 401 Unauthorized lease renewal: %v", err)return}- logger.Info("Successfully re-enrolled after 401 Unauthorized lease renewal, retrying lease renewal...")+ logger.Info("Successfully recovered after 401 Unauthorized lease renewal, retrying lease renewal...")continue}@@ -1232,10 +1249,14 @@ func (r *Router) RefreshEnrollment(ctx context.Context) error {return fmt.Errorf("failed to generate signature: %w", err)}- // 2. Construct request+ // 2. Construct request. peer_id lets the control plane find this+ // router's record when the biscuit's signing key has been retired and+ // the biscuit itself can no longer be verified (autonomous recovery,+ // opt-in server-side); it is cross-checked against the biscuit otherwise.req := &api.TokenRefreshRequest{ChallengeSignature: sig,Timestamp: timestamp,+ PeerId: peerID.String(),}reqData, err := proto.Marshal(req)if err != nil {diff --git a/internal/storage/round_trip_test.go b/internal/storage/round_trip_test.goindex e03c418d..07cb172e 100644--- a/internal/storage/round_trip_test.go+++ b/internal/storage/round_trip_test.go@@ -16,6 +16,7 @@ package storageimport ("context"+ "errors""reflect""strings""testing"@@ -119,6 +120,8 @@ func TestEnrolledNodeRoundTripsEveryField(t *testing.T) {Labels: map[string]string{"region": "emea"},EnrolledAt: time.Now().Add(-time.Hour),ExpiresAt: time.Now().Add(time.Hour),+ // Non-default so the round trip proves the column is read back.+ AutonomousRecovery: true,}// Banned is never set by enrollment; a node cannot un-ban itself by// re-enrolling. TestReEnrollmentKeepsAnExistingBan covers that.@@ -145,6 +148,51 @@ func TestEnrolledNodeRoundTripsEveryField(t *testing.T) {requireFieldsRoundTrip(t, want, &listed[0], "Banned")}+// Autonomous recovery is a per-node, server-side opt-in: it must default+// off, flip on and off via SetNodeAutonomousRecovery, and report ErrNotFound+// for a peer that was never enrolled (so an admin toggle can 404 instead of+// silently updating zero rows).+func TestSetNodeAutonomousRecovery(t *testing.T) {+ store := newTestStore(t)+ ctx := context.Background()++ node := &EnrolledNode{+ PeerID: "12D3KooWRecover",+ PublicKey: []byte("pub"),+ Biscuit: []byte("biscuit"),+ Role: api.RoleRouter,+ EnrollmentType: "BOOTSTRAP",+ EnrolledAt: time.Now(),+ }+ if err := store.EnrollNode(ctx, node); err != nil {+ t.Fatalf("EnrollNode: %v", err)+ }+ got, err := store.GetNode(ctx, node.PeerID)+ if err != nil {+ t.Fatalf("GetNode: %v", err)+ }+ if got.AutonomousRecovery {+ t.Fatal("AutonomousRecovery must default to false")+ }++ if err := store.SetNodeAutonomousRecovery(ctx, node.PeerID, true); err != nil {+ t.Fatalf("SetNodeAutonomousRecovery(true): %v", err)+ }+ if got, _ = store.GetNode(ctx, node.PeerID); !got.AutonomousRecovery {+ t.Fatal("SetNodeAutonomousRecovery(true) did not persist")+ }+ if err := store.SetNodeAutonomousRecovery(ctx, node.PeerID, false); err != nil {+ t.Fatalf("SetNodeAutonomousRecovery(false): %v", err)+ }+ if got, _ = store.GetNode(ctx, node.PeerID); got.AutonomousRecovery {+ t.Fatal("SetNodeAutonomousRecovery(false) did not persist")+ }++ if err := store.SetNodeAutonomousRecovery(ctx, "12D3KooWNobody", true); !errors.Is(err, ErrNotFound) {+ t.Fatalf("SetNodeAutonomousRecovery(unknown peer) = %v, want ErrNotFound", err)+ }+}+// A ban is the mesh's way of turning a node off. Re-enrolling must not clear// it, or revocation would last only until the node asked again.func TestReEnrollmentKeepsAnExistingBan(t *testing.T) {@@ -201,6 +249,8 @@ func TestBootstrapTokenRoundTripsEveryField(t *testing.T) {Description: "a description",CreatedAt: time.Now().Add(-time.Hour),ExpiresAt: time.Now().Add(time.Hour),+ // Non-default so the round trip proves the column is read back.+ AutonomousRecovery: true,}// RevokedAt is never set at creation - only RevokeBootstrapToken sets it,// covered separately below.diff --git a/internal/storage/sql_store.go b/internal/storage/sql_store.goindex 5713ace6..b8d717d4 100644--- a/internal/storage/sql_store.go+++ b/internal/storage/sql_store.go@@ -381,6 +381,20 @@ var migrations = []migration{`ALTER TABLE bootstrap_tokens ADD COLUMN revoked_at BIGINT`,},},+ {+ // Opt-in autonomous recovery (#367): a per-node flag, seeded from the+ // enrolling bootstrap token, that lets /refresh re-issue a biscuit+ // whose signing key has been retired. Deny by default.+ version: 10,+ postgres: []string{+ `ALTER TABLE nodes ADD COLUMN IF NOT EXISTS autonomous_recovery BOOLEAN DEFAULT FALSE NOT NULL`,+ `ALTER TABLE bootstrap_tokens ADD COLUMN IF NOT EXISTS autonomous_recovery BOOLEAN DEFAULT FALSE NOT NULL`,+ },+ sqlite: []string{+ `ALTER TABLE nodes ADD COLUMN autonomous_recovery BOOLEAN DEFAULT FALSE NOT NULL`,+ `ALTER TABLE bootstrap_tokens ADD COLUMN autonomous_recovery BOOLEAN DEFAULT FALSE NOT NULL`,+ },+ },}func (s *SQLStore) initSchema() error {@@ -633,16 +647,16 @@ func (s *SQLStore) EnrollNode(ctx context.Context, node *EnrolledNode) error {var query stringif s.isPostgres() {query = s.rebind(`- INSERT INTO nodes (peer_id, public_key, biscuit_token, role, enrollment_type, claims_json, owner_id, labels_json, enrolled_at, expires_at, banned)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, FALSE)+ INSERT INTO nodes (peer_id, public_key, biscuit_token, role, enrollment_type, claims_json, owner_id, labels_json, enrolled_at, expires_at, autonomous_recovery, banned)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, FALSE)ON CONFLICT (peer_id)- DO UPDATE SET public_key = EXCLUDED.public_key, biscuit_token = EXCLUDED.biscuit_token, role = EXCLUDED.role, enrollment_type = EXCLUDED.enrollment_type, claims_json = EXCLUDED.claims_json, owner_id = EXCLUDED.owner_id, labels_json = EXCLUDED.labels_json, enrolled_at = EXCLUDED.enrolled_at, expires_at = EXCLUDED.expires_at`)+ DO UPDATE SET public_key = EXCLUDED.public_key, biscuit_token = EXCLUDED.biscuit_token, role = EXCLUDED.role, enrollment_type = EXCLUDED.enrollment_type, claims_json = EXCLUDED.claims_json, owner_id = EXCLUDED.owner_id, labels_json = EXCLUDED.labels_json, enrolled_at = EXCLUDED.enrolled_at, expires_at = EXCLUDED.expires_at, autonomous_recovery = EXCLUDED.autonomous_recovery`)} else {query = s.rebind(`- INSERT INTO nodes (peer_id, public_key, biscuit_token, role, enrollment_type, claims_json, owner_id, labels_json, enrolled_at, expires_at, banned)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)+ INSERT INTO nodes (peer_id, public_key, biscuit_token, role, enrollment_type, claims_json, owner_id, labels_json, enrolled_at, expires_at, autonomous_recovery, banned)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)ON CONFLICT (peer_id)- DO UPDATE SET public_key = excluded.public_key, biscuit_token = excluded.biscuit_token, role = excluded.role, enrollment_type = excluded.enrollment_type, claims_json = excluded.claims_json, owner_id = excluded.owner_id, labels_json = excluded.labels_json, enrolled_at = excluded.enrolled_at, expires_at = excluded.expires_at`)+ DO UPDATE SET public_key = excluded.public_key, biscuit_token = excluded.biscuit_token, role = excluded.role, enrollment_type = excluded.enrollment_type, claims_json = excluded.claims_json, owner_id = excluded.owner_id, labels_json = excluded.labels_json, enrolled_at = excluded.enrolled_at, expires_at = excluded.expires_at, autonomous_recovery = excluded.autonomous_recovery`)}ownerIDNull := sql.NullString{String: node.OwnerID, Valid: node.OwnerID != ""}@@ -657,13 +671,14 @@ func (s *SQLStore) EnrollNode(ctx context.Context, node *EnrolledNode) error {string(labelsJSON),node.EnrolledAt.UnixMilli(),node.ExpiresAt.UnixMilli(),+ node.AutonomousRecovery,)return err}// GetNode implements Store.func (s *SQLStore) GetNode(ctx context.Context, peerID string) (*EnrolledNode, error) {- query := s.rebind(`SELECT peer_id, public_key, biscuit_token, role, enrollment_type, claims_json, owner_id, labels_json, enrolled_at, expires_at, banned FROM nodes WHERE peer_id = ?`)+ query := s.rebind(`SELECT peer_id, public_key, biscuit_token, role, enrollment_type, claims_json, owner_id, labels_json, enrolled_at, expires_at, banned, autonomous_recovery FROM nodes WHERE peer_id = ?`)var node EnrolledNodevar claimsJSON, ownerID, labelsJSON sql.NullStringvar enrolledAtUnix, expiresAtUnix int64@@ -679,6 +694,7 @@ func (s *SQLStore) GetNode(ctx context.Context, peerID string) (*EnrolledNode, e&enrolledAtUnix,&expiresAtUnix,&node.Banned,+ &node.AutonomousRecovery,)if claimsJSON.Valid {node.ClaimsJSON = claimsJSON.String@@ -709,6 +725,19 @@ func (s *SQLStore) SetNodeBanned(ctx context.Context, peerID string, banned boolreturn err}+// SetNodeAutonomousRecovery implements Store.+func (s *SQLStore) SetNodeAutonomousRecovery(ctx context.Context, peerID string, enabled bool) error {+ query := s.rebind(`UPDATE nodes SET autonomous_recovery = ? WHERE peer_id = ?`)+ res, err := s.db.ExecContext(ctx, query, enabled, peerID)+ if err != nil {+ return err+ }+ if n, err := res.RowsAffected(); err == nil && n == 0 {+ return ErrNotFound+ }+ return nil+}+// IsNodeBanned implements Store.func (s *SQLStore) IsNodeBanned(ctx context.Context, peerID string) (bool, error) {query := s.rebind(`SELECT banned FROM nodes WHERE peer_id = ?`)@@ -996,13 +1025,13 @@ func (s *SQLStore) SaveBootstrapToken(ctx context.Context, token *BootstrapTokenvar query stringif s.isPostgres() {query = s.rebind(`- INSERT INTO bootstrap_tokens (id, token_hash, role, owner_id, max_usages, usages_count, description, created_at, expires_at)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)+ INSERT INTO bootstrap_tokens (id, token_hash, role, owner_id, max_usages, usages_count, description, created_at, expires_at, autonomous_recovery)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)ON CONFLICT (id) DO NOTHING`)} else {query = s.rebind(`- INSERT INTO bootstrap_tokens (id, token_hash, role, owner_id, max_usages, usages_count, description, created_at, expires_at)- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)+ INSERT INTO bootstrap_tokens (id, token_hash, role, owner_id, max_usages, usages_count, description, created_at, expires_at, autonomous_recovery)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)ON CONFLICT (id) DO NOTHING`)}ownerIDNull := sql.NullString{String: token.OwnerID, Valid: token.OwnerID != ""}@@ -1016,13 +1045,14 @@ func (s *SQLStore) SaveBootstrapToken(ctx context.Context, token *BootstrapTokentoken.Description,token.CreatedAt.Unix(),token.ExpiresAt.Unix(),+ token.AutonomousRecovery,)return err}// GetBootstrapToken retrieves a bootstrap token by its ID (sha256 hash).func (s *SQLStore) GetBootstrapToken(ctx context.Context, id string) (*BootstrapToken, error) {- query := s.rebind(`SELECT id, token_hash, role, owner_id, max_usages, usages_count, description, created_at, expires_at, revoked_at FROM bootstrap_tokens WHERE id = ?`)+ query := s.rebind(`SELECT id, token_hash, role, owner_id, max_usages, usages_count, description, created_at, expires_at, revoked_at, autonomous_recovery FROM bootstrap_tokens WHERE id = ?`)var t BootstrapTokenvar created, expires int64var ownerID sql.NullString@@ -1038,6 +1068,7 @@ func (s *SQLStore) GetBootstrapToken(ctx context.Context, id string) (*Bootstrap&created,&expires,&revokedAt,+ &t.AutonomousRecovery,)if err == sql.ErrNoRows {return nil, ErrNotFound@@ -1217,7 +1248,7 @@ func (s *SQLStore) UpdateEnrollmentRequest(ctx context.Context, id string, statu// ListNodes retrieves all enrolled nodes.func (s *SQLStore) ListNodes(ctx context.Context) ([]EnrolledNode, error) {- query := s.rebind(`SELECT peer_id, public_key, biscuit_token, role, enrollment_type, claims_json, owner_id, labels_json, enrolled_at, expires_at, banned FROM nodes ORDER BY enrolled_at DESC`)+ query := s.rebind(`SELECT peer_id, public_key, biscuit_token, role, enrollment_type, claims_json, owner_id, labels_json, enrolled_at, expires_at, banned, autonomous_recovery FROM nodes ORDER BY enrolled_at DESC`)rows, err := s.db.QueryContext(ctx, query)if err != nil {return nil, fmt.Errorf("failed to query nodes: %w", err)@@ -1241,6 +1272,7 @@ func (s *SQLStore) ListNodes(ctx context.Context) ([]EnrolledNode, error) {&enrolledAtUnix,&expiresAtUnix,&node.Banned,+ &node.AutonomousRecovery,)if err != nil {return nil, fmt.Errorf("failed to scan enrolled node: %w", err)@@ -1277,7 +1309,7 @@ func (s *SQLStore) ListNodes(ctx context.Context) ([]EnrolledNode, error) {// ListBootstrapTokens retrieves all bootstrap tokens.func (s *SQLStore) ListBootstrapTokens(ctx context.Context) ([]BootstrapToken, error) {- query := s.rebind(`SELECT id, token_hash, role, owner_id, max_usages, usages_count, description, created_at, expires_at, revoked_at FROM bootstrap_tokens ORDER BY created_at DESC`)+ query := s.rebind(`SELECT id, token_hash, role, owner_id, max_usages, usages_count, description, created_at, expires_at, revoked_at, autonomous_recovery FROM bootstrap_tokens ORDER BY created_at DESC`)rows, err := s.db.QueryContext(ctx, query)if err != nil {return nil, fmt.Errorf("failed to query bootstrap tokens: %w", err)@@ -1302,6 +1334,7 @@ func (s *SQLStore) ListBootstrapTokens(ctx context.Context) ([]BootstrapToken, e&created,&expires,&revokedAt,+ &t.AutonomousRecovery,)if err != nil {return nil, fmt.Errorf("failed to scan bootstrap token: %w", err)diff --git a/internal/storage/storage.go b/internal/storage/storage.goindex 623e6dc6..2f6e599f 100644--- a/internal/storage/storage.go+++ b/internal/storage/storage.go@@ -72,6 +72,13 @@ type EnrolledNode struct {EnrolledAt time.TimeExpiresAt time.TimeBanned bool+ // AutonomousRecovery lets /refresh re-issue this node's biscuit after the+ // key that signed it has been retired, on proof of possession alone. Off+ // by default: without it a bootstrap node offline past the key grace+ // period needs an operator to mint a new token, which is what stops a+ // forgotten or stolen machine from rejoining on its own. Only ever set+ // server-side, from the token that enrolled the node or by an admin.+ AutonomousRecovery bool}// CheckAdmission reports whether the control plane may still serve this node.@@ -105,6 +112,9 @@ type BootstrapToken struct {// into this table and a hard delete would break that trail. Nil means// never revoked.RevokedAt *time.Time+ // AutonomousRecovery is copied onto every node this token enrolls; see+ // EnrolledNode.AutonomousRecovery.+ AutonomousRecovery bool}// IsRevoked reports whether the token has been explicitly revoked, as@@ -180,6 +190,10 @@ type Store interface {// IsNodeBanned checks if a node is currently banned.IsNodeBanned(ctx context.Context, peerID string) (bool, error)+ // SetNodeAutonomousRecovery toggles EnrolledNode.AutonomousRecovery.+ // Returns ErrNotFound if no such node is enrolled.+ SetNodeAutonomousRecovery(ctx context.Context, peerID string, enabled bool) error+// SetIdentityBanned bans or unbans an enrolled OIDC identity, keyed as// "issuer|subject". A node ban is keyed on a self-generated peer id, so// this is what makes a ban survive keypair regeneration.diff --git a/site/content/docs/user/control-plane-configuration.md b/site/content/docs/user/control-plane-configuration.mdindex ffabae5f..047df9ec 100644--- a/site/content/docs/user/control-plane-configuration.md+++ b/site/content/docs/user/control-plane-configuration.md@@ -26,7 +26,7 @@ The Control Plane is responsible for bridging user identities from trusted OIDC| `--biscuit-ttl` | *None* | `24h` | Lifespan minted into every issued Biscuit token. Capped to the OIDC token's own expiry when shorter. || `--oidc-session-ttl` | *None* | `2160h` (90 days) | How long an OIDC enrollment stays refreshable before the identity must re-authenticate with the OIDC provider. Shorter values keep the provider authoritative for offboarding, at the cost of more frequent interactive re-enrollment. || `--key-rotation-interval` | *None* | `24h` | Key rotation interval (e.g. `24h`). `0s` disables rotation. |-| `--key-grace-period` | *None* | `1h` | Key grace period for rotated keys. |+| `--key-grace-period` | *None* | `1h` | How long a rotated-out signing key stays accepted. Once it is retired, Biscuits it signed can no longer be verified or refreshed; see [Signing-Key Retirement and Recovery](#signing-key-retirement-and-recovery). || `--lease-duration` | *None* | `15m` | Router lease registration TTL. |---@@ -175,7 +175,7 @@ To enforce centralized access control without sacrificing offline verification pAll minted Biscuit tokens are cryptographically bound to a strict 24-hour expiration. Peers verify this expiration locally without hitting the Control Plane.2. **Long-Lived Sessions (The Right to Refresh)**:* **OIDC Interactive Enrollment**: 90-day database session limit. After 90 days, the user must re-enroll interactively.- * **Bootstrap Flow (Headless Nodes/Routers)**: Infinite session limit (sessions never expire).+ * **Bootstrap Flow (Headless Nodes/Routers)**: Infinite session limit (sessions never expire). In practice the signing key's grace period bounds how long such a node can stay offline and still refresh; see [Signing-Key Retirement and Recovery](#signing-key-retirement-and-recovery).### Proactive Refresh Lifecycle@@ -188,6 +188,44 @@ The bootstrap surface requires the same proof of possession end to end. `POST /eAll three challenges share one shape — the UTF-8 bytes of `sam:<endpoint>:<peer_id>:<unix-millis>` (`sam:enroll:…`, `sam:enroll-status:…`, `sam:refresh:…`), signed by the peer's identity key and accepted within a ±5-minute freshness window. Binding the peer and the endpoint into the signed payload means a signature captured from any one request verifies nowhere else.+### Signing-Key Retirement and Recovery++Every `--key-rotation-interval` the Control Plane mints a new signing key. The previous key stays accepted for `--key-grace-period`, then it is retired: Biscuits it signed can no longer be verified by anyone, including the Control Plane. A node whose Biscuit is signed by a retired key is refused on `/refresh` (`401`), fails its role check at daemon start, and — for a router — has its lease renewal rejected. This is deliberate: the grace period is the deadline after which a node that has gone quiet cannot come back on its own, so a forgotten or stolen machine does not rejoin the mesh unnoticed.++How a node gets past that deadline depends on how it was enrolled.++* **OIDC nodes** recover on their own. The daemon exchanges its stored refresh grant for a fresh ID token and re-enrolls under the current key, keeping its PeerID. The identity provider stays authoritative: revoking the grant there ends this.+* **Bootstrap nodes and routers** (headless enrollment) have no such grant. By default they need an operator: mint a new bootstrap token and re-run `sam-node join` (or restart the router with the new token). `POST /enroll` for a peer that is already approved re-mints a fresh Biscuit under the active key instead of replaying the stored one — the role and labels come from the enrolled node record, never from the request, the peer must still prove possession of its key, a banned peer is refused, and the re-enrollment consumes one use of the token. The token's `max_usages` is therefore the operator's cap on how many times a given node can be brought back this way.++#### Autonomous recovery (opt-in)++For fleets where an operator round-trip per stale node is impractical, a bootstrap node can be allowed to recover on proof of possession alone. This is a per-node flag, `autonomous_recovery`, held only on the Control Plane's node record and never asserted by the node. It is **off by default**: a node carrying it can always renew on its own private key, so that key effectively becomes a credential that never expires. Only a ban (`POST /admin/revoke`) stops it.++Set it either when minting the bootstrap token — every node that token enrolls inherits it — or per node afterwards:++```bash+# At mint time (admin only): nodes enrolled with this token may recover autonomously+curl -X POST -H "Authorization: Bearer <your-admin-token>" -H "Content-Type: application/json" \+ -d '{"role": "sam:role:router", "ttl_hours": 24, "max_usages": 3, "autonomous_recovery": true}' \+ http://<control-plane-ip>:8080/admin/bootstrap-tokens++# Per node, for one that is already enrolled (toggle back with "enabled": false)+curl -X POST -H "Authorization: Bearer <your-admin-token>" -H "Content-Type: application/json" \+ -d '{"enabled": true}' \+ http://<control-plane-ip>:8080/admin/nodes/<peer-id>/autonomous-recovery+```++The console shows the flag in the **Recovery** column of the Nodes and Bootstrap Tokens views, with an enable/disable action for administrators.++With the flag set, `/refresh` accepts a Biscuit it can no longer verify when all of the following hold:++1. the request carries the node's `peer_id`, and an enrolled record exists for it with `autonomous_recovery` on;+2. the presented Biscuit is byte-for-byte the last one the Control Plane issued to that node — a replayed, superseded Biscuit is refused, and this is what authenticates the Biscuit in place of the signature that cannot be checked;+3. the fresh challenge is signed by the node's registered private key;+4. the node is not banned and its session has not expired.++The new Biscuit is minted under the active key with the role and labels from the node record. Until steps 1–3 all succeed, a request through this path is answered exactly like any other bad Biscuit, so the fallback does not reveal which peer IDs are enrolled. Nodes and routers send `peer_id` on every refresh; when the Biscuit verifies normally it is only cross-checked against the token.+### Administrative RevocationAdministrators can immediately revoke any active session to disable a node's ability to renew its token.@@ -241,6 +279,15 @@ This returns a JSON response containing the plaintext token:}```+The request also accepts `"autonomous_recovery": true` (admin only), which lets every node enrolled with the token refresh its Biscuit after the signing key that issued it has been retired; see [Autonomous recovery](#autonomous-recovery-opt-in) before turning it on.++A token that is no longer wanted — a multi-use token being decommissioned early, or one that leaked — can be revoked before it expires. Revocation is a soft state on the token (it stays listed, marked revoked, so the audit trail of what it enrolled is kept), is idempotent, and is enforced on every `/enroll`, including re-enrollment of a node it previously approved:++```bash+curl -X DELETE -H "Authorization: Bearer <your-admin-token>" \+ http://<control-plane-ip>:8080/admin/bootstrap-tokens/<token-id>+```+### Step 2: Request Enrollment on the NodeRun the node `join` command with the generated token:@@ -270,3 +317,7 @@ curl -X POST \Alternatively, you can boot the control plane with `--auto-approve-enrollment` to automatically approve all valid bootstrap token requests without manual gates.+### Re-enrolling a node that is already approved++Running `sam-node join` again for a peer the Control Plane has already approved — after `sam-node reset`, or because its Biscuit was signed by a key that has since been retired — does not go back through the approval queue. As long as the token is valid, unrevoked and has usages left, the peer proves possession of the same key, its role matches the token's, and it is not banned, `POST /enroll` mints a fresh Biscuit under the current signing key from the stored node record and returns it immediately. This is the manual recovery path described in [Signing-Key Retirement and Recovery](#signing-key-retirement-and-recovery); no database intervention is needed.+