Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
feat(node): recover a stale identity at startup via the refresh token
Antonio Ojeab888389cebd2
diff --git a/internal/node/enroll.go b/internal/node/enroll.goindex 3cc6b461..f3e9807b 100644--- a/internal/node/enroll.go+++ b/internal/node/enroll.go@@ -29,6 +29,7 @@ import ("github.com/google/sam/internal/identity"golog "github.com/ipfs/go-log/v2""github.com/libp2p/go-libp2p/core/crypto"+ "github.com/libp2p/go-libp2p/core/peer""github.com/multiformats/go-multiaddr""google.golang.org/protobuf/proto")@@ -62,39 +63,50 @@ func GetOrGenerateKey(s *Store) crypto.PrivKey {func (n *SamNode) Enroll(ctx context.Context, controlPlaneURL string, jwt string) error {pubKey := n.Host.Peerstore().PubKey(n.Host.ID())+ enrollResp, err := n.enrollHTTP(ctx, controlPlaneURL, jwt, n.Host.ID(), pubKey)+ if err != nil {+ return err+ }++ return n.connectToRouters(ctx, enrollResp.RouterAddresses)+}++// enrollHTTP performs the HTTP half of enrollment for an explicit peer+// identity, so it can run before the libp2p host exists (startup recovery).+func (n *SamNode) enrollHTTP(ctx context.Context, controlPlaneURL, jwt string, peerID peer.ID, pubKey crypto.PubKey) (*api.EnrollResponse, error) {pubBytes, err := crypto.MarshalPublicKey(pubKey)if err != nil {- return fmt.Errorf("failed to marshal public key: %w", err)+ return nil, fmt.Errorf("failed to marshal public key: %w", err)}req := &api.EnrollRequest{Jwt: jwt,- PeerId: n.Host.ID().String(),+ PeerId: peerID.String(),PublicKey: pubBytes,RequestedRole: n.config.RequiredRole,Labels: n.config.Labels, // validated at startup}data, err := proto.Marshal(req)if err != nil {- return fmt.Errorf("failed to marshal enroll request: %v", err)+ return nil, fmt.Errorf("failed to marshal enroll request: %v", err)}if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {- return fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)+ return nil, fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)}url := controlPlaneURL + "/register"logger.Infof("Enrolling via HTTP at %s", url)httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(data))if err != nil {- return fmt.Errorf("failed to create HTTP request: %v", err)+ return nil, fmt.Errorf("failed to create HTTP request: %v", err)}httpReq.Header.Set("Content-Type", "application/x-protobuf")client := &http.Client{Timeout: 30 * time.Second}resp, err := client.Do(httpReq)if err != nil {- return fmt.Errorf("HTTP request failed: %v", err)+ return nil, fmt.Errorf("HTTP request failed: %v", err)}defer func() {if err := resp.Body.Close(); err != nil {@@ -102,12 +114,29 @@ func (n *SamNode) Enroll(ctx context.Context, controlPlaneURL string, jwt string}}()- enrollResp, err := n.processEnrollResponse(resp)+ return n.processEnrollResponse(resp)+}++// recoverStaleIdentity re-enrolls over HTTP with a JWT obtained from the+// stored refresh token, keeping the node's PeerID. It must not touch n.Host:+// it runs before the host exists, and router connection happens in the+// normal startup path afterwards.+func (n *SamNode) recoverStaleIdentity(ctx context.Context) error {+ controlPlaneURL, err := n.Store.LoadControlPlaneURL()+ if err != nil || controlPlaneURL == "" {+ return fmt.Errorf("no control plane URL in store")+ }+ jwt, err := n.renewWithRefreshToken(ctx, "")if err != nil {return err}-- return n.connectToRouters(ctx, enrollResp.RouterAddresses)+ privKey := GetOrGenerateKey(n.Store)+ peerID, err := peer.IDFromPublicKey(privKey.GetPublic())+ if err != nil {+ return fmt.Errorf("failed to derive peer ID from stored key: %w", err)+ }+ _, err = n.enrollHTTP(ctx, controlPlaneURL, jwt, peerID, privKey.GetPublic())+ return err}func (n *SamNode) processEnrollResponse(resp *http.Response) (*api.EnrollResponse, error) {diff --git a/internal/node/enroll_test.go b/internal/node/enroll_test.goindex 7c1276ee..f80ee015 100644--- a/internal/node/enroll_test.go+++ b/internal/node/enroll_test.go@@ -17,18 +17,214 @@ package nodeimport ("bytes""context"+ "crypto/ed25519"+ "encoding/json""io""net/http""net/http/httptest""strings"+ "sync""testing""time"+ "github.com/biscuit-auth/biscuit-go/v2""github.com/google/sam/api"+ "github.com/google/sam/internal/identity""github.com/libp2p/go-libp2p/core/crypto"+ "github.com/libp2p/go-libp2p/core/peer""google.golang.org/protobuf/proto")+// TestStartRecoversStaleIdentityViaRefreshToken covers #321: a stored+// identity signed by a fully rotated-out key must heal at startup through+// refresh grant + HTTP re-enrollment, keeping the PeerID, instead of fataling.+func TestStartRecoversStaleIdentityViaRefreshToken(t *testing.T) {+ cpPub, cpPriv, _ := ed25519.GenerateKey(nil)+ _, stalePriv, _ := ed25519.GenerateKey(nil)++ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = store.Close() }()++ // Fix the node keypair up front so the mock CP can assert PeerID is kept+ privKey := GetOrGenerateKey(store)+ wantPeerID, err := peer.IDFromPublicKey(privKey.GetPublic())+ if err != nil {+ t.Fatal(err)+ }++ mint := func(priv ed25519.PrivateKey, peerID string) []byte {+ builder := biscuit.NewBuilder(priv)+ for _, f := range []biscuit.Fact{+ {Predicate: biscuit.Predicate{Name: api.FactNode, IDs: []biscuit.Term{biscuit.String(peerID)}}},+ {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(24 * time.Hour))}}},+ } {+ if err := builder.AddAuthorityFact(f); err != nil {+ t.Fatalf("failed to add fact: %v", err)+ }+ }+ tok, err := builder.Build()+ if err != nil {+ t.Fatalf("failed to build biscuit: %v", err)+ }+ b, err := tok.Serialize()+ if err != nil {+ t.Fatalf("failed to serialize biscuit: %v", err)+ }+ return b+ }++ // Stored identity is signed by a key the node no longer trusts+ if err := store.SaveIdentity(mint(stalePriv, wantPeerID.String())); err != nil {+ t.Fatal(err)+ }++ // Mock OIDC issuer honouring the refresh grant+ oidcMux := http.NewServeMux()+ oidcMux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(map[string]any{+ "issuer": "http://" + r.Host,+ "token_endpoint": "http://" + r.Host + "/token",+ "authorization_endpoint": "http://" + r.Host + "/auth",+ })+ })+ oidcMux.HandleFunc("/token", func(w http.ResponseWriter, r *http.Request) {+ _ = r.ParseForm()+ if r.FormValue("grant_type") != "refresh_token" || r.FormValue("refresh_token") != "stored_refresh" {+ http.Error(w, "invalid grant", http.StatusBadRequest)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(map[string]string{+ "access_token": "unused",+ "id_token": "recovered_jwt",+ "refresh_token": "rotated_refresh",+ })+ })+ oidcSrv := httptest.NewServer(oidcMux)+ defer oidcSrv.Close()++ // Mock control plane: re-enrolls the peer with a freshly signed biscuit+ var mu sync.Mutex+ var gotPeerID, gotJWT string+ cpMux := http.NewServeMux()+ cpMux.HandleFunc("/register", func(w http.ResponseWriter, r *http.Request) {+ body, _ := io.ReadAll(r.Body)+ var req api.EnrollRequest+ if err := proto.Unmarshal(body, &req); err != nil {+ http.Error(w, "bad request", http.StatusBadRequest)+ return+ }+ mu.Lock()+ gotPeerID, gotJWT = req.PeerId, req.Jwt+ mu.Unlock()+ resp := &api.EnrollResponse{+ BiscuitToken: mint(cpPriv, req.PeerId),+ ControlPlanePublicKey: cpPub,+ RouterAddresses: []string{"/ip4/127.0.0.1/tcp/1"},+ Expiration: time.Now().Add(24 * time.Hour).Unix(),+ }+ data, _ := proto.Marshal(resp)+ w.Header().Set("Content-Type", "application/x-protobuf")+ _, _ = w.Write(data)+ })+ cpSrv := httptest.NewServer(cpMux)+ defer cpSrv.Close()++ if err := store.SaveOIDCConfig(oidcSrv.URL, "client_id_test", "sam"); err != nil {+ t.Fatal(err)+ }+ if err := store.SaveRefreshToken("stored_refresh"); err != nil {+ t.Fatal(err)+ }+ if err := store.SaveControlPlaneURL(cpSrv.URL); err != nil {+ t.Fatal(err)+ }++ node, err := NewSamNode(Options{+ PrivKey: privKey,+ Store: store,+ ControlPlanePubKey: cpPub,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ })+ if err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ if err := node.Start(ctx); err != nil {+ t.Fatalf("Start should have recovered the stale identity, got: %v", err)+ }++ mu.Lock()+ if gotPeerID != wantPeerID.String() {+ t.Errorf("re-enrollment peer id: got %q, want %q (PeerID must survive recovery)", gotPeerID, wantPeerID)+ }+ if gotJWT != "recovered_jwt" {+ t.Errorf("re-enrollment JWT: got %q, want refresh-grant token", gotJWT)+ }+ mu.Unlock()++ if err := identity.VerifyBiscuitRole(node.GetIdentity(), cpPub, api.RoleNode, time.Second); err != nil {+ t.Errorf("recovered identity does not verify under the current CP key: %v", err)+ }+}++// Without a refresh token the startup failure must stay fatal.+func TestStartStaleIdentityWithoutRefreshTokenFails(t *testing.T) {+ cpPub, _, _ := ed25519.GenerateKey(nil)+ _, stalePriv, _ := ed25519.GenerateKey(nil)++ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = store.Close() }()++ privKey := GetOrGenerateKey(store)+ peerID, _ := peer.IDFromPublicKey(privKey.GetPublic())++ builder := biscuit.NewBuilder(stalePriv)+ _ = builder.AddAuthorityFact(biscuit.Fact{Predicate: biscuit.Predicate{Name: api.FactNode, IDs: []biscuit.Term{biscuit.String(peerID.String())}}})+ _ = builder.AddAuthorityFact(biscuit.Fact{Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}})+ tok, err := builder.Build()+ if err != nil {+ t.Fatal(err)+ }+ staleBiscuit, err := tok.Serialize()+ if err != nil {+ t.Fatal(err)+ }+ if err := store.SaveIdentity(staleBiscuit); err != nil {+ t.Fatal(err)+ }++ node, err := NewSamNode(Options{+ PrivKey: privKey,+ Store: store,+ ControlPlanePubKey: cpPub,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ })+ if err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()++ err = node.Start(ctx)+ if err == nil {+ t.Fatal("Start should fail when recovery is impossible")+ }+ if !strings.Contains(err.Error(), "fails role requirement") {+ t.Errorf("error should keep the role-requirement guidance, got: %v", err)+ }+}+func TestGetOrGenerateKey(t *testing.T) {dir := t.TempDir()store, err := NewStore(dir)diff --git a/internal/node/node.go b/internal/node/node.goindex 4634f05a..85d12e71 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -324,7 +324,14 @@ func (n *SamNode) Start(ctx context.Context) error {}}if roleErr != nil {- return fmt.Errorf("loaded identity fails role requirement %q: %w", n.config.RequiredRole, roleErr)+ // Stale identity (e.g. signing key rotated past its grace+ // period while offline): try silent re-enrollment with the+ // stored refresh token before giving up.+ logger.Warnf("Loaded identity fails role requirement %q: %v; attempting recovery via stored refresh token", n.config.RequiredRole, roleErr)+ if recErr := n.recoverStaleIdentity(ctx); recErr != nil {+ return fmt.Errorf("loaded identity fails role requirement %q (refresh-token recovery failed: %v): %w", n.config.RequiredRole, recErr, roleErr)+ }+ logger.Info("Identity recovered via refresh-token re-enrollment.")}}}