Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: enroll again with the bootstrap token when the control plane forgets the node
Antonio Ojea857d4c6460b2
diff --git a/cmd/agentmesh-node/main.go b/cmd/agentmesh-node/main.goindex 9609dfcc..1867847b 100644--- a/cmd/agentmesh-node/main.go+++ b/cmd/agentmesh-node/main.go@@ -463,6 +463,7 @@ func main() {RouterConnectTimeout: routerConnectTimeoutFlag,RequiredRole: api.RoleNode,ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,+ BootstrapTokenPath: bootstrapTokenPathFlag,DHTLookupLimit: dhtLookupLimitFlag,DiscoveryConcurrency: discoveryConcurrencyFlag,BackendProbeTimeout: backendProbeTimeoutFlag,@@ -758,6 +759,7 @@ func main() {RouterConnectTimeout: routerConnectTimeoutFlag,RequiredRole: api.RoleNode,ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,+ BootstrapTokenPath: bootstrapTokenPathFlag,})if err != nil {logger.Fatalf("Failed to initialize node for enrollment: %v", err)diff --git a/internal/controlplane/client/client.go b/internal/controlplane/client/client.goindex fd906a8e..fbbe9f6e 100644--- a/internal/controlplane/client/client.go+++ b/internal/controlplane/client/client.go@@ -56,6 +56,23 @@ var ErrBodyTooLarge = errors.New("control plane answer exceeds the body cap")// release check for it, so a newer node works against an older control plane.var ErrNotFound = errors.New("control plane does not serve this endpoint")+// StatusError is a control plane answer other than success, with the status+// code a caller can act on (a 401 on an authenticated endpoint says the+// control plane does not know this node any more) and the body it sent.+type StatusError struct {+ Code int+ Status string+ Body string+}++func (e *StatusError) Error() string {+ return fmt.Sprintf("control plane returned status %s: %s", e.Status, e.Body)+}++func newStatusError(resp *http.Response, body []byte) *StatusError {+ return &StatusError{Code: resp.StatusCode, Status: resp.Status, Body: strings.TrimSpace(string(body))}+}+// ReadBody reads a control plane response body of at most MaxBodyBytes and// reports ErrBodyTooLarge for anything larger.func ReadBody(r io.Reader) ([]byte, error) {@@ -269,7 +286,7 @@ func (c *Client) ReportCatalog(ctx context.Context, biscuit []byte, services []*if resp.StatusCode != http.StatusNoContent {body, _ := ReadBody(resp.Body)- return fmt.Errorf("control plane returned status %s: %s", resp.Status, string(body))+ return newStatusError(resp, body)}return nil}@@ -351,7 +368,7 @@ func (c *Client) get(ctx context.Context, path string, biscuit []byte, challengereturn fmt.Errorf("%w: %s", ErrNotFound, path)}if resp.StatusCode != http.StatusOK {- return fmt.Errorf("control plane returned status %s: %s", resp.Status, string(body))+ return newStatusError(resp, body)}if err := proto.Unmarshal(body, msg); err != nil {return fmt.Errorf("failed to decode %s response: %w", path, err)@@ -392,7 +409,7 @@ func (c *Client) postWithChallenge(ctx context.Context, path string, biscuit []breturn fmt.Errorf("%w: %s", ErrNotFound, path)}if resp.StatusCode != http.StatusOK {- return fmt.Errorf("control plane returned status %s: %s", resp.Status, strings.TrimSpace(string(body)))+ return newStatusError(resp, body)}if err := proto.Unmarshal(body, respMsg); err != nil {return fmt.Errorf("failed to decode %s response: %w", path, err)diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.goindex 741cc15b..20334c3d 100644--- a/internal/node/controlplane_sync.go+++ b/internal/node/controlplane_sync.go@@ -20,11 +20,16 @@ import ("errors""fmt""math/rand"+ "net/http"+ "os"+ "strings""time"cpclient "github.com/google/agentmesh/internal/controlplane/client""github.com/libp2p/go-libp2p/core/peer""github.com/multiformats/go-multiaddr"+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto")// The node reads three things from the control plane while it runs: the@@ -328,6 +333,9 @@ func (n *AgentMeshNode) startControlPlaneSyncLoop(ctx context.Context, intervaltimer.Reset(interval + time.Duration(rand.Int63n(int64(interval/10)+1)))err := n.SyncControlPlane(ctx)+ if err != nil && n.controlPlaneForgotUs(err) && n.recoverEnrollment(ctx) {+ err = n.SyncControlPlane(ctx)+ }switch {case err != nil && failures == 0:logger.Warnf("Control plane sync failed: %v", err)@@ -344,3 +352,65 @@ func (n *AgentMeshNode) startControlPlaneSyncLoop(ctx context.Context, interval}}()}++// reenrollmentsTotal counts the times this node enrolled again on its own+// after the control plane stopped recognizing its credential.+var reenrollmentsTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "agentmesh_node_reenrollments_total",+ Help: "Times this node enrolled again with its bootstrap token after the control plane answered 401 to an authenticated call, by result",+ },+ []string{"result"},+)++// controlPlaneForgotUs reports whether err is the control plane refusing this+// node's credential outright (401 on an authenticated endpoint). A control+// plane that lost its store, such as a new Cloud Run revision without a+// database, answers that to a node whose Biscuit still verifies everywhere+// else: the signing key is the same, the enrollment record is gone.+func (n *AgentMeshNode) controlPlaneForgotUs(err error) bool {+ var se *cpclient.StatusError+ return errors.As(err, &se) && se.Code == http.StatusUnauthorized+}++// recoverEnrollment enrolls again with the bootstrap token at+// Options.BootstrapTokenPath and reports whether it did. Without a path the+// node leaves recovery to the renewal loop and says so once per failure.+func (n *AgentMeshNode) recoverEnrollment(ctx context.Context) bool {+ path := n.config.BootstrapTokenPath+ if path == "" {+ logger.Warn("Control plane no longer recognizes this node's credential and no --bootstrap-token-path is configured; waiting for the renewal loop")+ reenrollmentsTotal.WithLabelValues("no_token").Inc()+ return false+ }+ raw, err := os.ReadFile(path)+ if err != nil {+ logger.Warnw("Control plane no longer recognizes this node's credential; bootstrap token unreadable", "path", path, "error", err)+ reenrollmentsTotal.WithLabelValues("no_token").Inc()+ return false+ }+ controlPlaneURL, err := n.Store.LoadControlPlaneURL()+ if err != nil {+ logger.Warnw("Control plane no longer recognizes this node's credential; control plane URL unknown", "error", err)+ reenrollmentsTotal.WithLabelValues("failed").Inc()+ return false+ }+ logger.Warnw("Control plane no longer recognizes this node's credential; enrolling again with the bootstrap token", "control_plane", controlPlaneURL)+ // The credential half only: this node is already attached to its+ // routers, and its own loop keeps it so.+ enroll := n.enrollBootstrapFn+ if enroll == nil {+ enroll = func(ctx context.Context, controlPlaneURL, token string) error {+ _, err := n.enrollBootstrapCredential(ctx, controlPlaneURL, token)+ return err+ }+ }+ if err := enroll(ctx, controlPlaneURL, strings.TrimSpace(string(raw))); err != nil {+ logger.Errorw("Re-enrollment with the bootstrap token failed", "error", err)+ reenrollmentsTotal.WithLabelValues("failed").Inc()+ return false+ }+ logger.Info("Re-enrolled with the bootstrap token; the control plane knows this node again")+ reenrollmentsTotal.WithLabelValues("ok").Inc()+ return true+}diff --git a/internal/node/controlplane_sync_test.go b/internal/node/controlplane_sync_test.gonew file mode 100644index 00000000..b246193e--- /dev/null+++ b/internal/node/controlplane_sync_test.go@@ -0,0 +1,119 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "context"+ "crypto/ed25519"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "sync/atomic"+ "testing"++ "github.com/google/agentmesh/api"+)++// TestSyncRecoversWhenControlPlaneForgotNode is the stateless control plane+// case: the signing key is unchanged, so this node's Biscuit still verifies+// everywhere, but the control plane lost its enrollment record and answers+// 401 to the node's own calls. A node with a bootstrap token file enrolls+// again at once and the same sync pass succeeds; without one it says so and+// leaves recovery to the renewal loop.+func TestSyncRecoversWhenControlPlaneForgotNode(t *testing.T) {+ newHarness := func(t *testing.T) (*refreshHarness, *atomic.Bool) {+ t.Helper()+ var known atomic.Bool // the control plane has this node's record+ harness := newRefreshHarness(t, func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "unexpected refresh", http.StatusInternalServerError)+ })+ mux := http.NewServeMux()+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{harness.cpPub}, []ed25519.PrivateKey{harness.cpPriv}))+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{}))+ mux.HandleFunc("/policies", func(w http.ResponseWriter, r *http.Request) {+ if !known.Load() {+ http.Error(w, "Unauthorized: node credential required", http.StatusUnauthorized)+ return+ }+ protoHandler(t, &api.PolicyConfigGetResponse{})(w, r)+ })+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)+ if err := harness.node.Store.SaveControlPlaneURL(server.URL); err != nil {+ t.Fatal(err)+ }+ return harness, &known+ }++ t.Run("re-enrolls with the bootstrap token", func(t *testing.T) {+ harness, known := newHarness(t)+ tokenPath := filepath.Join(t.TempDir(), "join-token")+ if err := os.WriteFile(tokenPath, []byte("mesh_tok_test\n"), 0o600); err != nil {+ t.Fatal(err)+ }+ harness.node.config.BootstrapTokenPath = tokenPath+ var enrolledWith string+ harness.node.enrollBootstrapFn = func(ctx context.Context, controlPlaneURL, token string) error {+ enrolledWith = token+ known.Store(true)+ return nil+ }++ err := harness.node.SyncControlPlane(context.Background())+ if !harness.node.controlPlaneForgotUs(err) {+ t.Fatalf("sync error = %v, want a 401 the node recognizes as a forgotten enrollment", err)+ }+ if !harness.node.recoverEnrollment(context.Background()) {+ t.Fatal("recoverEnrollment returned false with a readable bootstrap token")+ }+ if enrolledWith != "mesh_tok_test" {+ t.Fatalf("enrolled with %q, want the trimmed token from the file", enrolledWith)+ }+ if err := harness.node.SyncControlPlane(context.Background()); err != nil {+ t.Fatalf("sync after re-enrollment: %v", err)+ }+ })++ t.Run("without a token path it waits for the renewal loop", func(t *testing.T) {+ harness, _ := newHarness(t)+ called := false+ harness.node.enrollBootstrapFn = func(context.Context, string, string) error { called = true; return nil }+ err := harness.node.SyncControlPlane(context.Background())+ if !harness.node.controlPlaneForgotUs(err) {+ t.Fatalf("sync error = %v, want a recognized 401", err)+ }+ if harness.node.recoverEnrollment(context.Background()) {+ t.Fatal("recoverEnrollment returned true without a bootstrap token path")+ }+ if called {+ t.Fatal("enrollment attempted without a token")+ }+ })++ t.Run("other failures are not a forgotten enrollment", func(t *testing.T) {+ harness, _ := newHarness(t)+ if err := harness.node.Store.SaveControlPlaneURL("http://127.0.0.1:1"); err != nil {+ t.Fatal(err)+ }+ err := harness.node.SyncControlPlane(context.Background())+ if err == nil {+ t.Fatal("sync against a closed port succeeded")+ }+ if harness.node.controlPlaneForgotUs(err) {+ t.Fatalf("a connection failure (%v) was taken for a 401", err)+ }+ })+}diff --git a/internal/node/enroll.go b/internal/node/enroll.goindex c07d5ac8..32d51a64 100644--- a/internal/node/enroll.go+++ b/internal/node/enroll.go@@ -261,14 +261,37 @@ func (n *AgentMeshNode) joinRouters(ctx context.Context, controlPlaneURL string,// EnrollBootstrap enrolls the node with the control plane using a pre-shared bootstrap token.// If the enrollment status is PENDING, it polls the status endpoint until approved or rejected.func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, bootstrapToken string) error {+ enrollResp, err := n.enrollBootstrapCredential(ctx, controlPlaneURL, bootstrapToken)+ if err != nil {+ return err+ }++ // Connect and Auth to router after enrollment to join the mesh+ if len(enrollResp.RouterAddresses) == 0 {+ return fmt.Errorf("failed to connect and authenticate after bootstrap enrollment: control plane returned no router addresses")+ }+ if err := n.joinRouters(ctx, controlPlaneURL, enrollResp.RouterAddresses); err != nil {+ return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment: %w", err)+ }++ logger.Info("Successfully enrolled via Bootstrap token and joined mesh.")+ return nil+}++// enrollBootstrapCredential is the HTTP half of EnrollBootstrap: it obtains+// a credential with the bootstrap token and persists it with the mesh+// configuration, without attaching to routers. A node that is already on+// the mesh and only needs the control plane to know it again (the control+// plane lost its store) uses this half alone.+func (n *AgentMeshNode) enrollBootstrapCredential(ctx context.Context, controlPlaneURL string, bootstrapToken string) (*api.BootstrapEnrollResponse, error) {pubKey := n.Host.Peerstore().PubKey(n.Host.ID())pubBytes, err := crypto.MarshalPublicKey(pubKey)if err != nil {- return fmt.Errorf("failed to marshal public key: %w", err)+ return nil, fmt.Errorf("failed to marshal public key: %w", err)}if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {- return fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)+ return nil, fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)}enrollURL := controlPlaneURL + "/enroll"logger.Infof("Enrolling via Bootstrap token at %s", enrollURL)@@ -300,27 +323,27 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL strreturn httpReq, nil})if err != nil {- return fmt.Errorf("HTTP request failed: %w", err)+ return nil, fmt.Errorf("HTTP request failed: %w", err)}defer func() { _ = resp.Body.Close() }()if resp.StatusCode != http.StatusOK {body, _ := io.ReadAll(io.LimitReader(resp.Body, maxControlPlaneBodyBytes))- return fmt.Errorf("enrollment failed with status %s: %s", resp.Status, string(body))+ return nil, fmt.Errorf("enrollment failed with status %s: %s", resp.Status, string(body))}respData, err := cpclient.ReadBody(resp.Body)if err != nil {- return fmt.Errorf("bootstrap enrollment response: %w", err)+ return nil, fmt.Errorf("bootstrap enrollment response: %w", err)}enrollResp := &api.BootstrapEnrollResponse{}if err := proto.Unmarshal(respData, enrollResp); err != nil {- return fmt.Errorf("failed to unmarshal response: %w", err)+ return nil, fmt.Errorf("failed to unmarshal response: %w", err)}if enrollResp.ErrorMessage != "" {- return fmt.Errorf("enrollment failed: %s", enrollResp.ErrorMessage)+ return nil, fmt.Errorf("enrollment failed: %s", enrollResp.ErrorMessage)}if enrollResp.Status == api.EnrollmentStatus_ENROLLMENT_STATUS_PENDING {@@ -328,7 +351,7 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL stru, err := url.Parse(controlPlaneURL)if err != nil {- return fmt.Errorf("invalid control plane URL: %w", err)+ return nil, fmt.Errorf("invalid control plane URL: %w", err)}u = u.JoinPath("enroll", "status")q := u.Query()@@ -342,7 +365,7 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL strfor {select {case <-ctx.Done():- return ctx.Err()+ return nil, ctx.Err()case <-ticker.C:// Prove possession of the enrollment key on every poll; the// control plane returns the biscuit only to the enrollee.@@ -382,54 +405,44 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL str}if statusResp.ErrorMessage != "" {- return fmt.Errorf("enrollment failed: %s", statusResp.ErrorMessage)+ return nil, fmt.Errorf("enrollment failed: %s", statusResp.ErrorMessage)}if statusResp.Status == api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {enrollResp = statusRespbreak pollLoop} else if statusResp.Status == api.EnrollmentStatus_ENROLLMENT_STATUS_REJECTED {- return fmt.Errorf("enrollment was rejected by administrator")+ return nil, fmt.Errorf("enrollment was rejected by administrator")}}}}if len(enrollResp.BiscuitToken) == 0 {- return fmt.Errorf("received empty biscuit token from enrollment response")+ return nil, fmt.Errorf("received empty biscuit token from enrollment response")}if len(enrollResp.ControlPlanePublicKey) != ed25519.PublicKeySize {- return fmt.Errorf("received invalid control plane public key size: %d bytes", len(enrollResp.ControlPlanePublicKey))+ return nil, fmt.Errorf("received invalid control plane public key size: %d bytes", len(enrollResp.ControlPlanePublicKey))}if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole, n.BiscuitTimeout); err != nil {- return fmt.Errorf("enrolled biscuit token lacks required role %q: %w", n.config.RequiredRole, err)+ return nil, fmt.Errorf("enrolled biscuit token lacks required role %q: %w", n.config.RequiredRole, err)}if err := n.Store.SaveIdentity(enrollResp.BiscuitToken); err != nil {- return fmt.Errorf("failed to save identity: %v", err)+ return nil, fmt.Errorf("failed to save identity: %v", err)}n.SetIdentityCache(enrollResp.BiscuitToken)if err := n.Store.SaveIdentityExpiration(expireUnix(enrollResp.ExpireTime)); err != nil {- return fmt.Errorf("failed to save identity expiration: %v", err)+ return nil, fmt.Errorf("failed to save identity expiration: %v", err)}if err := n.Store.SaveMeshConfig(enrollResp.ControlPlanePublicKey, enrollResp.RouterAddresses); err != nil {- return fmt.Errorf("failed to save mesh config: %v", err)+ return nil, fmt.Errorf("failed to save mesh config: %v", err)}n.adoptEnrolledKeys(ctx, controlPlaneURL, enrollResp.ControlPlanePublicKey)-- // Connect and Auth to router after enrollment to join the mesh- if len(enrollResp.RouterAddresses) == 0 {- return fmt.Errorf("failed to connect and authenticate after bootstrap enrollment: control plane returned no router addresses")- }- if err := n.joinRouters(ctx, controlPlaneURL, enrollResp.RouterAddresses); err != nil {- return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment: %w", err)- }-- logger.Info("Successfully enrolled via Bootstrap token and joined mesh.")- return nil+ return enrollResp, nil}diff --git a/internal/node/node.go b/internal/node/node.goindex 8be81b97..a2294b34 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -144,7 +144,9 @@ func (n *AgentMeshNode) isAdmitted(p peer.ID) bool {}type AgentMeshNode struct {- config Options+ config Options+ // enrollBootstrapFn is EnrollBootstrap unless a test injects the step.+ enrollBootstrapFn func(ctx context.Context, controlPlaneURL, bootstrapToken string) errorHost host.HostDirectory DirectoryPubSub *pubsub.PubSubdiff --git a/internal/node/options.go b/internal/node/options.goindex d2565f14..d400eaf0 100644--- a/internal/node/options.go+++ b/internal/node/options.go@@ -142,6 +142,13 @@ type Options struct {// with the fleet's cadence, or a policy update on a large mesh is a// stampede.ControlPlaneSyncJitter time.Duration+ // BootstrapTokenPath is the file the node's bootstrap (join) token is+ // read from. A node that has it can enroll again on its own when the+ // control plane answers 401 to an authenticated call, which is what a+ // control plane that lost its store (a new Cloud Run revision without a+ // database) does; the token is read again each time so a rotated token+ // is picked up. Empty leaves recovery to the renewal loop.+ BootstrapTokenPath string// BackendProbeTimeout bounds how long a command-spawned service backend// (agentmesh-node.yaml's `command`, spawned as a local subprocess) is given to// answer before the service is registered but withheld fromdiff --git a/site/content/docs/guides/cloud-run.md b/site/content/docs/guides/cloud-run.mdindex 93e2f688..0138d19a 100644--- a/site/content/docs/guides/cloud-run.md+++ b/site/content/docs/guides/cloud-run.md@@ -83,11 +83,12 @@ file, keep the previous one until every credential it signed has expired(the Biscuit TTL for Ed25519, the STS JWT lifetime for P-256), publish the newsecret version, roll a revision, then drop the old key the same way.-What this does not keep is the enrollment records: a node's own calls to the+What this does not keep is the enrollment records. A node's own calls to thecontrol plane (policy sync, catalog, refresh) answer `401 node credential-required` after a revision until the node re-enrolls, which it does at its-next renewal with the bootstrap token. The data plane is unaffected in the-meantime. PostgreSQL keeps the records too.+required` after a revision; a node started with `--bootstrap-token-path`+notices that at its next sync and enrolls again with the token, without a+restart, and the data plane is unaffected in the meantime. PostgreSQL keeps+the records too.Why each Cloud Run flag matters:diff --git a/site/content/docs/reference/agentmesh-node.md b/site/content/docs/reference/agentmesh-node.mdindex 84b50be8..c2849832 100644--- a/site/content/docs/reference/agentmesh-node.md+++ b/site/content/docs/reference/agentmesh-node.md@@ -58,7 +58,7 @@ bootstrap token, `join` runs an interactive OIDC login.| Flag | Default | Meaning ||---|---|---|-| `--bootstrap-token-path` | | File containing a bootstrap token. |+| `--bootstrap-token-path` | | File containing a bootstrap token. A node that keeps this file can also recover on its own: when the control plane answers `401` to one of its authenticated calls (a control plane that lost its store, such as a new Cloud Run revision without a database), the node reads the token again and enrolls again at the next sync, without a restart; `agentmesh_node_reenrollments_total{result}` counts it. || `--bootstrap-token` | | The token as a value. Visible in process listings. The file form is preferred. || `--offline-access` | `false` | Request an OIDC refresh token, so that the node can re-enroll on its own when its credential expires. || `--allow-loopback` | `false` | Advertise and dial loopback and link-local addresses. For several nodes on one host. |