Skip to content

Agent Mesh

feat(node): recover a stale identity at startup via the refresh token

Antonio Ojeab888389cebd2
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index 3cc6b461..f3e9807b 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -29,6 +29,7 @@ import (
"github.com/google/sam/internal/identity"
golog "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
"github.com/multiformats/go-multiaddr"
"google.golang.org/protobuf/proto"
)
@@ -62,39 +63,50 @@ func GetOrGenerateKey(s *Store) crypto.PrivKey {
func (n *SamNode) Enroll(ctx context.Context, controlPlaneURL string, jwt string) error {
pubKey := n.Host.Peerstore().PubKey(n.Host.ID())
+ enrollResp, err := n.enrollHTTP(ctx, controlPlaneURL, jwt, n.Host.ID(), pubKey)
+ if err != nil {
+ return err
+ }
+
+ return n.connectToRouters(ctx, enrollResp.RouterAddresses)
+}
+
+// enrollHTTP performs the HTTP half of enrollment for an explicit peer
+// identity, so it can run before the libp2p host exists (startup recovery).
+func (n *SamNode) enrollHTTP(ctx context.Context, controlPlaneURL, jwt string, peerID peer.ID, pubKey crypto.PubKey) (*api.EnrollResponse, error) {
pubBytes, err := crypto.MarshalPublicKey(pubKey)
if err != nil {
- return fmt.Errorf("failed to marshal public key: %w", err)
+ return nil, fmt.Errorf("failed to marshal public key: %w", err)
}
req := &api.EnrollRequest{
Jwt: jwt,
- PeerId: n.Host.ID().String(),
+ PeerId: peerID.String(),
PublicKey: pubBytes,
RequestedRole: n.config.RequiredRole,
Labels: n.config.Labels, // validated at startup
}
data, err := proto.Marshal(req)
if err != nil {
- return fmt.Errorf("failed to marshal enroll request: %v", err)
+ return nil, fmt.Errorf("failed to marshal enroll request: %v", err)
}
if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {
- return fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)
+ return nil, fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)
}
url := controlPlaneURL + "/register"
logger.Infof("Enrolling via HTTP at %s", url)
httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(data))
if err != nil {
- return fmt.Errorf("failed to create HTTP request: %v", err)
+ return nil, fmt.Errorf("failed to create HTTP request: %v", err)
}
httpReq.Header.Set("Content-Type", "application/x-protobuf")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(httpReq)
if err != nil {
- return fmt.Errorf("HTTP request failed: %v", err)
+ return nil, fmt.Errorf("HTTP request failed: %v", err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
@@ -102,12 +114,29 @@ func (n *SamNode) Enroll(ctx context.Context, controlPlaneURL string, jwt string
}
}()
- enrollResp, err := n.processEnrollResponse(resp)
+ return n.processEnrollResponse(resp)
+}
+
+// recoverStaleIdentity re-enrolls over HTTP with a JWT obtained from the
+// stored refresh token, keeping the node's PeerID. It must not touch n.Host:
+// it runs before the host exists, and router connection happens in the
+// normal startup path afterwards.
+func (n *SamNode) recoverStaleIdentity(ctx context.Context) error {
+ controlPlaneURL, err := n.Store.LoadControlPlaneURL()
+ if err != nil || controlPlaneURL == "" {
+ return fmt.Errorf("no control plane URL in store")
+ }
+ jwt, err := n.renewWithRefreshToken(ctx, "")
if err != nil {
return err
}
-
- return n.connectToRouters(ctx, enrollResp.RouterAddresses)
+ privKey := GetOrGenerateKey(n.Store)
+ peerID, err := peer.IDFromPublicKey(privKey.GetPublic())
+ if err != nil {
+ return fmt.Errorf("failed to derive peer ID from stored key: %w", err)
+ }
+ _, err = n.enrollHTTP(ctx, controlPlaneURL, jwt, peerID, privKey.GetPublic())
+ return err
}
func (n *SamNode) processEnrollResponse(resp *http.Response) (*api.EnrollResponse, error) {
diff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.go
index 7c1276ee..f80ee015 100644
--- a/internal/node/enroll_test.go
+++ b/internal/node/enroll_test.go
@@ -17,18 +17,214 @@ package node
import (
"bytes"
"context"
+ "crypto/ed25519"
+ "encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
+ "sync"
"testing"
"time"
+ "github.com/biscuit-auth/biscuit-go/v2"
"github.com/google/sam/api"
+ "github.com/google/sam/internal/identity"
"github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
"google.golang.org/protobuf/proto"
)
+// TestStartRecoversStaleIdentityViaRefreshToken covers #321: a stored
+// identity signed by a fully rotated-out key must heal at startup through
+// refresh grant + HTTP re-enrollment, keeping the PeerID, instead of fataling.
+func TestStartRecoversStaleIdentityViaRefreshToken(t *testing.T) {
+ cpPub, cpPriv, _ := ed25519.GenerateKey(nil)
+ _, stalePriv, _ := ed25519.GenerateKey(nil)
+
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+
+ // Fix the node keypair up front so the mock CP can assert PeerID is kept
+ privKey := GetOrGenerateKey(store)
+ wantPeerID, err := peer.IDFromPublicKey(privKey.GetPublic())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ mint := func(priv ed25519.PrivateKey, peerID string) []byte {
+ builder := biscuit.NewBuilder(priv)
+ for _, f := range []biscuit.Fact{
+ {Predicate: biscuit.Predicate{Name: api.FactNode, IDs: []biscuit.Term{biscuit.String(peerID)}}},
+ {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},
+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(24 * time.Hour))}}},
+ } {
+ if err := builder.AddAuthorityFact(f); err != nil {
+ t.Fatalf("failed to add fact: %v", err)
+ }
+ }
+ tok, err := builder.Build()
+ if err != nil {
+ t.Fatalf("failed to build biscuit: %v", err)
+ }
+ b, err := tok.Serialize()
+ if err != nil {
+ t.Fatalf("failed to serialize biscuit: %v", err)
+ }
+ return b
+ }
+
+ // Stored identity is signed by a key the node no longer trusts
+ if err := store.SaveIdentity(mint(stalePriv, wantPeerID.String())); err != nil {
+ t.Fatal(err)
+ }
+
+ // Mock OIDC issuer honouring the refresh grant
+ oidcMux := http.NewServeMux()
+ oidcMux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "issuer": "http://" + r.Host,
+ "token_endpoint": "http://" + r.Host + "/token",
+ "authorization_endpoint": "http://" + r.Host + "/auth",
+ })
+ })
+ oidcMux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {
+ _ = r.ParseForm()
+ if r.FormValue("grant_type") != "refresh_token" || r.FormValue("refresh_token") != "stored_refresh" {
+ http.Error(w, "invalid grant", http.StatusBadRequest)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(map[string]string{
+ "access_token": "unused",
+ "id_token": "recovered_jwt",
+ "refresh_token": "rotated_refresh",
+ })
+ })
+ oidcSrv := httptest.NewServer(oidcMux)
+ defer oidcSrv.Close()
+
+ // Mock control plane: re-enrolls the peer with a freshly signed biscuit
+ var mu sync.Mutex
+ var gotPeerID, gotJWT string
+ cpMux := http.NewServeMux()
+ cpMux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {
+ body, _ := io.ReadAll(r.Body)
+ var req api.EnrollRequest
+ if err := proto.Unmarshal(body, &req); err != nil {
+ http.Error(w, "bad request", http.StatusBadRequest)
+ return
+ }
+ mu.Lock()
+ gotPeerID, gotJWT = req.PeerId, req.Jwt
+ mu.Unlock()
+ resp := &api.EnrollResponse{
+ BiscuitToken: mint(cpPriv, req.PeerId),
+ ControlPlanePublicKey: cpPub,
+ RouterAddresses: []string{"/ip4/127.0.0.1/tcp/1"},
+ Expiration: time.Now().Add(24 * time.Hour).Unix(),
+ }
+ data, _ := proto.Marshal(resp)
+ w.Header().Set("Content-Type", "application/x-protobuf")
+ _, _ = w.Write(data)
+ })
+ cpSrv := httptest.NewServer(cpMux)
+ defer cpSrv.Close()
+
+ if err := store.SaveOIDCConfig(oidcSrv.URL, "client_id_test", "sam"); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveRefreshToken("stored_refresh"); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveControlPlaneURL(cpSrv.URL); err != nil {
+ t.Fatal(err)
+ }
+
+ node, err := NewSamNode(Options{
+ PrivKey: privKey,
+ Store: store,
+ ControlPlanePubKey: cpPub,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ if err := node.Start(ctx); err != nil {
+ t.Fatalf("Start should have recovered the stale identity, got: %v", err)
+ }
+
+ mu.Lock()
+ if gotPeerID != wantPeerID.String() {
+ t.Errorf("re-enrollment peer id: got %q, want %q (PeerID must survive recovery)", gotPeerID, wantPeerID)
+ }
+ if gotJWT != "recovered_jwt" {
+ t.Errorf("re-enrollment JWT: got %q, want refresh-grant token", gotJWT)
+ }
+ mu.Unlock()
+
+ if err := identity.VerifyBiscuitRole(node.GetIdentity(), cpPub, api.RoleNode, time.Second); err != nil {
+ t.Errorf("recovered identity does not verify under the current CP key: %v", err)
+ }
+}
+
+// Without a refresh token the startup failure must stay fatal.
+func TestStartStaleIdentityWithoutRefreshTokenFails(t *testing.T) {
+ cpPub, _, _ := ed25519.GenerateKey(nil)
+ _, stalePriv, _ := ed25519.GenerateKey(nil)
+
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = store.Close() }()
+
+ privKey := GetOrGenerateKey(store)
+ peerID, _ := peer.IDFromPublicKey(privKey.GetPublic())
+
+ builder := biscuit.NewBuilder(stalePriv)
+ _ = builder.AddAuthorityFact(biscuit.Fact{Predicate: biscuit.Predicate{Name: api.FactNode, IDs: []biscuit.Term{biscuit.String(peerID.String())}}})
+ _ = builder.AddAuthorityFact(biscuit.Fact{Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}})
+ tok, err := builder.Build()
+ if err != nil {
+ t.Fatal(err)
+ }
+ staleBiscuit, err := tok.Serialize()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := store.SaveIdentity(staleBiscuit); err != nil {
+ t.Fatal(err)
+ }
+
+ node, err := NewSamNode(Options{
+ PrivKey: privKey,
+ Store: store,
+ ControlPlanePubKey: cpPub,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ err = node.Start(ctx)
+ if err == nil {
+ t.Fatal("Start should fail when recovery is impossible")
+ }
+ if !strings.Contains(err.Error(), "fails role requirement") {
+ t.Errorf("error should keep the role-requirement guidance, got: %v", err)
+ }
+}
+
func TestGetOrGenerateKey(t *testing.T) {
dir := t.TempDir()
store, err := NewStore(dir)
diff --git a/internal/node/node.go b/internal/node/node.go
index 4634f05a..85d12e71 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -324,7 +324,14 @@ func (n *SamNode) Start(ctx context.Context) error {
}
}
if roleErr != nil {
- return fmt.Errorf("loaded identity fails role requirement %q: %w", n.config.RequiredRole, roleErr)
+ // Stale identity (e.g. signing key rotated past its grace
+ // period while offline): try silent re-enrollment with the
+ // stored refresh token before giving up.
+ logger.Warnf("Loaded identity fails role requirement %q: %v; attempting recovery via stored refresh token", n.config.RequiredRole, roleErr)
+ if recErr := n.recoverStaleIdentity(ctx); recErr != nil {
+ return fmt.Errorf("loaded identity fails role requirement %q (refresh-token recovery failed: %v): %w", n.config.RequiredRole, recErr, roleErr)
+ }
+ logger.Info("Identity recovered via refresh-token re-enrollment.")
}
}
}