Skip to content

Agent Mesh

node: enroll again with the bootstrap token when the control plane forgets the node

Antonio Ojea857d4c6460b2
diff --git a/cmd/agentmesh-node/main.go b/cmd/agentmesh-node/main.go
index 9609dfcc..1867847b 100644
--- a/cmd/agentmesh-node/main.go
+++ b/cmd/agentmesh-node/main.go
@@ -463,6 +463,7 @@ func main() {
RouterConnectTimeout: routerConnectTimeoutFlag,
RequiredRole: api.RoleNode,
ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,
+ BootstrapTokenPath: bootstrapTokenPathFlag,
DHTLookupLimit: dhtLookupLimitFlag,
DiscoveryConcurrency: discoveryConcurrencyFlag,
BackendProbeTimeout: backendProbeTimeoutFlag,
@@ -758,6 +759,7 @@ func main() {
RouterConnectTimeout: routerConnectTimeoutFlag,
RequiredRole: api.RoleNode,
ControlPlaneSyncInterval: controlPlaneSyncIntervalFlag,
+ BootstrapTokenPath: bootstrapTokenPathFlag,
})
if err != nil {
logger.Fatalf("Failed to initialize node for enrollment: %v", err)
diff --git a/internal/controlplane/client/client.go b/internal/controlplane/client/client.go
index fd906a8e..fbbe9f6e 100644
--- a/internal/controlplane/client/client.go
+++ b/internal/controlplane/client/client.go
@@ -56,6 +56,23 @@ var ErrBodyTooLarge = errors.New("control plane answer exceeds the body cap")
// release check for it, so a newer node works against an older control plane.
var ErrNotFound = errors.New("control plane does not serve this endpoint")
+// StatusError is a control plane answer other than success, with the status
+// code a caller can act on (a 401 on an authenticated endpoint says the
+// control plane does not know this node any more) and the body it sent.
+type StatusError struct {
+ Code int
+ Status string
+ Body string
+}
+
+func (e *StatusError) Error() string {
+ return fmt.Sprintf("control plane returned status %s: %s", e.Status, e.Body)
+}
+
+func newStatusError(resp *http.Response, body []byte) *StatusError {
+ return &StatusError{Code: resp.StatusCode, Status: resp.Status, Body: strings.TrimSpace(string(body))}
+}
+
// ReadBody reads a control plane response body of at most MaxBodyBytes and
// reports ErrBodyTooLarge for anything larger.
func ReadBody(r io.Reader) ([]byte, error) {
@@ -269,7 +286,7 @@ func (c *Client) ReportCatalog(ctx context.Context, biscuit []byte, services []*
if resp.StatusCode != http.StatusNoContent {
body, _ := ReadBody(resp.Body)
- return fmt.Errorf("control plane returned status %s: %s", resp.Status, string(body))
+ return newStatusError(resp, body)
}
return nil
}
@@ -351,7 +368,7 @@ func (c *Client) get(ctx context.Context, path string, biscuit []byte, challenge
return fmt.Errorf("%w: %s", ErrNotFound, path)
}
if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("control plane returned status %s: %s", resp.Status, string(body))
+ return newStatusError(resp, body)
}
if err := proto.Unmarshal(body, msg); err != nil {
return fmt.Errorf("failed to decode %s response: %w", path, err)
@@ -392,7 +409,7 @@ func (c *Client) postWithChallenge(ctx context.Context, path string, biscuit []b
return fmt.Errorf("%w: %s", ErrNotFound, path)
}
if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("control plane returned status %s: %s", resp.Status, strings.TrimSpace(string(body)))
+ return newStatusError(resp, body)
}
if err := proto.Unmarshal(body, respMsg); err != nil {
return fmt.Errorf("failed to decode %s response: %w", path, err)
diff --git a/internal/node/controlplane_sync.go b/internal/node/controlplane_sync.go
index 741cc15b..20334c3d 100644
--- a/internal/node/controlplane_sync.go
+++ b/internal/node/controlplane_sync.go
@@ -20,11 +20,16 @@ import (
"errors"
"fmt"
"math/rand"
+ "net/http"
+ "os"
+ "strings"
"time"
cpclient "github.com/google/agentmesh/internal/controlplane/client"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/multiformats/go-multiaddr"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
)
// The node reads three things from the control plane while it runs: the
@@ -328,6 +333,9 @@ func (n *AgentMeshNode) startControlPlaneSyncLoop(ctx context.Context, interval
timer.Reset(interval + time.Duration(rand.Int63n(int64(interval/10)+1)))
err := n.SyncControlPlane(ctx)
+ if err != nil && n.controlPlaneForgotUs(err) && n.recoverEnrollment(ctx) {
+ err = n.SyncControlPlane(ctx)
+ }
switch {
case err != nil && failures == 0:
logger.Warnf("Control plane sync failed: %v", err)
@@ -344,3 +352,65 @@ func (n *AgentMeshNode) startControlPlaneSyncLoop(ctx context.Context, interval
}
}()
}
+
+// reenrollmentsTotal counts the times this node enrolled again on its own
+// after the control plane stopped recognizing its credential.
+var reenrollmentsTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_node_reenrollments_total",
+ Help: "Times this node enrolled again with its bootstrap token after the control plane answered 401 to an authenticated call, by result",
+ },
+ []string{"result"},
+)
+
+// controlPlaneForgotUs reports whether err is the control plane refusing this
+// node's credential outright (401 on an authenticated endpoint). A control
+// plane that lost its store, such as a new Cloud Run revision without a
+// database, answers that to a node whose Biscuit still verifies everywhere
+// else: the signing key is the same, the enrollment record is gone.
+func (n *AgentMeshNode) controlPlaneForgotUs(err error) bool {
+ var se *cpclient.StatusError
+ return errors.As(err, &se) && se.Code == http.StatusUnauthorized
+}
+
+// recoverEnrollment enrolls again with the bootstrap token at
+// Options.BootstrapTokenPath and reports whether it did. Without a path the
+// node leaves recovery to the renewal loop and says so once per failure.
+func (n *AgentMeshNode) recoverEnrollment(ctx context.Context) bool {
+ path := n.config.BootstrapTokenPath
+ if path == "" {
+ logger.Warn("Control plane no longer recognizes this node's credential and no --bootstrap-token-path is configured; waiting for the renewal loop")
+ reenrollmentsTotal.WithLabelValues("no_token").Inc()
+ return false
+ }
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ logger.Warnw("Control plane no longer recognizes this node's credential; bootstrap token unreadable", "path", path, "error", err)
+ reenrollmentsTotal.WithLabelValues("no_token").Inc()
+ return false
+ }
+ controlPlaneURL, err := n.Store.LoadControlPlaneURL()
+ if err != nil {
+ logger.Warnw("Control plane no longer recognizes this node's credential; control plane URL unknown", "error", err)
+ reenrollmentsTotal.WithLabelValues("failed").Inc()
+ return false
+ }
+ logger.Warnw("Control plane no longer recognizes this node's credential; enrolling again with the bootstrap token", "control_plane", controlPlaneURL)
+ // The credential half only: this node is already attached to its
+ // routers, and its own loop keeps it so.
+ enroll := n.enrollBootstrapFn
+ if enroll == nil {
+ enroll = func(ctx context.Context, controlPlaneURL, token string) error {
+ _, err := n.enrollBootstrapCredential(ctx, controlPlaneURL, token)
+ return err
+ }
+ }
+ if err := enroll(ctx, controlPlaneURL, strings.TrimSpace(string(raw))); err != nil {
+ logger.Errorw("Re-enrollment with the bootstrap token failed", "error", err)
+ reenrollmentsTotal.WithLabelValues("failed").Inc()
+ return false
+ }
+ logger.Info("Re-enrolled with the bootstrap token; the control plane knows this node again")
+ reenrollmentsTotal.WithLabelValues("ok").Inc()
+ return true
+}
diff --git a/internal/node/controlplane_sync_test.go b/internal/node/controlplane_sync_test.go
new file mode 100644
index 00000000..b246193e
--- /dev/null
+++ b/internal/node/controlplane_sync_test.go
@@ -0,0 +1,119 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "crypto/ed25519"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "sync/atomic"
+ "testing"
+
+ "github.com/google/agentmesh/api"
+)
+
+// TestSyncRecoversWhenControlPlaneForgotNode is the stateless control plane
+// case: the signing key is unchanged, so this node's Biscuit still verifies
+// everywhere, but the control plane lost its enrollment record and answers
+// 401 to the node's own calls. A node with a bootstrap token file enrolls
+// again at once and the same sync pass succeeds; without one it says so and
+// leaves recovery to the renewal loop.
+func TestSyncRecoversWhenControlPlaneForgotNode(t *testing.T) {
+ newHarness := func(t *testing.T) (*refreshHarness, *atomic.Bool) {
+ t.Helper()
+ var known atomic.Bool // the control plane has this node's record
+ harness := newRefreshHarness(t, func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "unexpected refresh", http.StatusInternalServerError)
+ })
+ mux := http.NewServeMux()
+ mux.HandleFunc("/keys", keysHandler(t, []ed25519.PublicKey{harness.cpPub}, []ed25519.PrivateKey{harness.cpPriv}))
+ mux.HandleFunc("/info", protoHandler(t, &api.ControlPlaneInfoResponse{}))
+ mux.HandleFunc("/policies", func(w http.ResponseWriter, r *http.Request) {
+ if !known.Load() {
+ http.Error(w, "Unauthorized: node credential required", http.StatusUnauthorized)
+ return
+ }
+ protoHandler(t, &api.PolicyConfigGetResponse{})(w, r)
+ })
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+ if err := harness.node.Store.SaveControlPlaneURL(server.URL); err != nil {
+ t.Fatal(err)
+ }
+ return harness, &known
+ }
+
+ t.Run("re-enrolls with the bootstrap token", func(t *testing.T) {
+ harness, known := newHarness(t)
+ tokenPath := filepath.Join(t.TempDir(), "join-token")
+ if err := os.WriteFile(tokenPath, []byte("mesh_tok_test\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ harness.node.config.BootstrapTokenPath = tokenPath
+ var enrolledWith string
+ harness.node.enrollBootstrapFn = func(ctx context.Context, controlPlaneURL, token string) error {
+ enrolledWith = token
+ known.Store(true)
+ return nil
+ }
+
+ err := harness.node.SyncControlPlane(context.Background())
+ if !harness.node.controlPlaneForgotUs(err) {
+ t.Fatalf("sync error = %v, want a 401 the node recognizes as a forgotten enrollment", err)
+ }
+ if !harness.node.recoverEnrollment(context.Background()) {
+ t.Fatal("recoverEnrollment returned false with a readable bootstrap token")
+ }
+ if enrolledWith != "mesh_tok_test" {
+ t.Fatalf("enrolled with %q, want the trimmed token from the file", enrolledWith)
+ }
+ if err := harness.node.SyncControlPlane(context.Background()); err != nil {
+ t.Fatalf("sync after re-enrollment: %v", err)
+ }
+ })
+
+ t.Run("without a token path it waits for the renewal loop", func(t *testing.T) {
+ harness, _ := newHarness(t)
+ called := false
+ harness.node.enrollBootstrapFn = func(context.Context, string, string) error { called = true; return nil }
+ err := harness.node.SyncControlPlane(context.Background())
+ if !harness.node.controlPlaneForgotUs(err) {
+ t.Fatalf("sync error = %v, want a recognized 401", err)
+ }
+ if harness.node.recoverEnrollment(context.Background()) {
+ t.Fatal("recoverEnrollment returned true without a bootstrap token path")
+ }
+ if called {
+ t.Fatal("enrollment attempted without a token")
+ }
+ })
+
+ t.Run("other failures are not a forgotten enrollment", func(t *testing.T) {
+ harness, _ := newHarness(t)
+ if err := harness.node.Store.SaveControlPlaneURL("http://127.0.0.1:1"); err != nil {
+ t.Fatal(err)
+ }
+ err := harness.node.SyncControlPlane(context.Background())
+ if err == nil {
+ t.Fatal("sync against a closed port succeeded")
+ }
+ if harness.node.controlPlaneForgotUs(err) {
+ t.Fatalf("a connection failure (%v) was taken for a 401", err)
+ }
+ })
+}
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index c07d5ac8..32d51a64 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -261,14 +261,37 @@ func (n *AgentMeshNode) joinRouters(ctx context.Context, controlPlaneURL string,
// EnrollBootstrap enrolls the node with the control plane using a pre-shared bootstrap token.
// If the enrollment status is PENDING, it polls the status endpoint until approved or rejected.
func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, bootstrapToken string) error {
+ enrollResp, err := n.enrollBootstrapCredential(ctx, controlPlaneURL, bootstrapToken)
+ if err != nil {
+ return err
+ }
+
+ // Connect and Auth to router after enrollment to join the mesh
+ if len(enrollResp.RouterAddresses) == 0 {
+ return fmt.Errorf("failed to connect and authenticate after bootstrap enrollment: control plane returned no router addresses")
+ }
+ if err := n.joinRouters(ctx, controlPlaneURL, enrollResp.RouterAddresses); err != nil {
+ return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment: %w", err)
+ }
+
+ logger.Info("Successfully enrolled via Bootstrap token and joined mesh.")
+ return nil
+}
+
+// enrollBootstrapCredential is the HTTP half of EnrollBootstrap: it obtains
+// a credential with the bootstrap token and persists it with the mesh
+// configuration, without attaching to routers. A node that is already on
+// the mesh and only needs the control plane to know it again (the control
+// plane lost its store) uses this half alone.
+func (n *AgentMeshNode) enrollBootstrapCredential(ctx context.Context, controlPlaneURL string, bootstrapToken string) (*api.BootstrapEnrollResponse, error) {
pubKey := n.Host.Peerstore().PubKey(n.Host.ID())
pubBytes, err := crypto.MarshalPublicKey(pubKey)
if err != nil {
- return fmt.Errorf("failed to marshal public key: %w", err)
+ return nil, fmt.Errorf("failed to marshal public key: %w", err)
}
if !strings.HasPrefix(controlPlaneURL, "http://") && !strings.HasPrefix(controlPlaneURL, "https://") {
- return fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)
+ return nil, fmt.Errorf("control plane address must be an HTTP or HTTPS URL for enrollment: %s", controlPlaneURL)
}
enrollURL := controlPlaneURL + "/enroll"
logger.Infof("Enrolling via Bootstrap token at %s", enrollURL)
@@ -300,27 +323,27 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL str
return httpReq, nil
})
if err != nil {
- return fmt.Errorf("HTTP request failed: %w", err)
+ return nil, fmt.Errorf("HTTP request failed: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, maxControlPlaneBodyBytes))
- return fmt.Errorf("enrollment failed with status %s: %s", resp.Status, string(body))
+ return nil, fmt.Errorf("enrollment failed with status %s: %s", resp.Status, string(body))
}
respData, err := cpclient.ReadBody(resp.Body)
if err != nil {
- return fmt.Errorf("bootstrap enrollment response: %w", err)
+ return nil, fmt.Errorf("bootstrap enrollment response: %w", err)
}
enrollResp := &api.BootstrapEnrollResponse{}
if err := proto.Unmarshal(respData, enrollResp); err != nil {
- return fmt.Errorf("failed to unmarshal response: %w", err)
+ return nil, fmt.Errorf("failed to unmarshal response: %w", err)
}
if enrollResp.ErrorMessage != "" {
- return fmt.Errorf("enrollment failed: %s", enrollResp.ErrorMessage)
+ return nil, fmt.Errorf("enrollment failed: %s", enrollResp.ErrorMessage)
}
if enrollResp.Status == api.EnrollmentStatus_ENROLLMENT_STATUS_PENDING {
@@ -328,7 +351,7 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL str
u, err := url.Parse(controlPlaneURL)
if err != nil {
- return fmt.Errorf("invalid control plane URL: %w", err)
+ return nil, fmt.Errorf("invalid control plane URL: %w", err)
}
u = u.JoinPath("enroll", "status")
q := u.Query()
@@ -342,7 +365,7 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL str
for {
select {
case <-ctx.Done():
- return ctx.Err()
+ return nil, ctx.Err()
case <-ticker.C:
// Prove possession of the enrollment key on every poll; the
// control plane returns the biscuit only to the enrollee.
@@ -382,54 +405,44 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL str
}
if statusResp.ErrorMessage != "" {
- return fmt.Errorf("enrollment failed: %s", statusResp.ErrorMessage)
+ return nil, fmt.Errorf("enrollment failed: %s", statusResp.ErrorMessage)
}
if statusResp.Status == api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {
enrollResp = statusResp
break pollLoop
} else if statusResp.Status == api.EnrollmentStatus_ENROLLMENT_STATUS_REJECTED {
- return fmt.Errorf("enrollment was rejected by administrator")
+ return nil, fmt.Errorf("enrollment was rejected by administrator")
}
}
}
}
if len(enrollResp.BiscuitToken) == 0 {
- return fmt.Errorf("received empty biscuit token from enrollment response")
+ return nil, fmt.Errorf("received empty biscuit token from enrollment response")
}
if len(enrollResp.ControlPlanePublicKey) != ed25519.PublicKeySize {
- return fmt.Errorf("received invalid control plane public key size: %d bytes", len(enrollResp.ControlPlanePublicKey))
+ return nil, fmt.Errorf("received invalid control plane public key size: %d bytes", len(enrollResp.ControlPlanePublicKey))
}
if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole, n.BiscuitTimeout); err != nil {
- return fmt.Errorf("enrolled biscuit token lacks required role %q: %w", n.config.RequiredRole, err)
+ return nil, fmt.Errorf("enrolled biscuit token lacks required role %q: %w", n.config.RequiredRole, err)
}
if err := n.Store.SaveIdentity(enrollResp.BiscuitToken); err != nil {
- return fmt.Errorf("failed to save identity: %v", err)
+ return nil, fmt.Errorf("failed to save identity: %v", err)
}
n.SetIdentityCache(enrollResp.BiscuitToken)
if err := n.Store.SaveIdentityExpiration(expireUnix(enrollResp.ExpireTime)); err != nil {
- return fmt.Errorf("failed to save identity expiration: %v", err)
+ return nil, fmt.Errorf("failed to save identity expiration: %v", err)
}
if err := n.Store.SaveMeshConfig(enrollResp.ControlPlanePublicKey, enrollResp.RouterAddresses); err != nil {
- return fmt.Errorf("failed to save mesh config: %v", err)
+ return nil, fmt.Errorf("failed to save mesh config: %v", err)
}
n.adoptEnrolledKeys(ctx, controlPlaneURL, enrollResp.ControlPlanePublicKey)
-
- // Connect and Auth to router after enrollment to join the mesh
- if len(enrollResp.RouterAddresses) == 0 {
- return fmt.Errorf("failed to connect and authenticate after bootstrap enrollment: control plane returned no router addresses")
- }
- if err := n.joinRouters(ctx, controlPlaneURL, enrollResp.RouterAddresses); err != nil {
- return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment: %w", err)
- }
-
- logger.Info("Successfully enrolled via Bootstrap token and joined mesh.")
- return nil
+ return enrollResp, nil
}
diff --git a/internal/node/node.go b/internal/node/node.go
index 8be81b97..a2294b34 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -144,7 +144,9 @@ func (n *AgentMeshNode) isAdmitted(p peer.ID) bool {
}
type AgentMeshNode struct {
- config Options
+ config Options
+ // enrollBootstrapFn is EnrollBootstrap unless a test injects the step.
+ enrollBootstrapFn func(ctx context.Context, controlPlaneURL, bootstrapToken string) error
Host host.Host
Directory Directory
PubSub *pubsub.PubSub
diff --git a/internal/node/options.go b/internal/node/options.go
index d2565f14..d400eaf0 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -142,6 +142,13 @@ type Options struct {
// with the fleet's cadence, or a policy update on a large mesh is a
// stampede.
ControlPlaneSyncJitter time.Duration
+ // BootstrapTokenPath is the file the node's bootstrap (join) token is
+ // read from. A node that has it can enroll again on its own when the
+ // control plane answers 401 to an authenticated call, which is what a
+ // control plane that lost its store (a new Cloud Run revision without a
+ // database) does; the token is read again each time so a rotated token
+ // is picked up. Empty leaves recovery to the renewal loop.
+ BootstrapTokenPath string
// BackendProbeTimeout bounds how long a command-spawned service backend
// (agentmesh-node.yaml's `command`, spawned as a local subprocess) is given to
// answer before the service is registered but withheld from
diff --git a/site/content/docs/guides/cloud-run.md b/site/content/docs/guides/cloud-run.md
index 93e2f688..0138d19a 100644
--- a/site/content/docs/guides/cloud-run.md
+++ b/site/content/docs/guides/cloud-run.md
@@ -83,11 +83,12 @@ file, keep the previous one until every credential it signed has expired
(the Biscuit TTL for Ed25519, the STS JWT lifetime for P-256), publish the new
secret version, roll a revision, then drop the old key the same way.
-What this does not keep is the enrollment records: a node's own calls to the
+What this does not keep is the enrollment records. A node's own calls to the
control plane (policy sync, catalog, refresh) answer `401 node credential
-required` after a revision until the node re-enrolls, which it does at its
-next renewal with the bootstrap token. The data plane is unaffected in the
-meantime. PostgreSQL keeps the records too.
+required` after a revision; a node started with `--bootstrap-token-path`
+notices that at its next sync and enrolls again with the token, without a
+restart, and the data plane is unaffected in the meantime. PostgreSQL keeps
+the records too.
Why each Cloud Run flag matters:
diff --git a/site/content/docs/reference/agentmesh-node.md b/site/content/docs/reference/agentmesh-node.md
index 84b50be8..c2849832 100644
--- a/site/content/docs/reference/agentmesh-node.md
+++ b/site/content/docs/reference/agentmesh-node.md
@@ -58,7 +58,7 @@ bootstrap token, `join` runs an interactive OIDC login.
| Flag | Default | Meaning |
|---|---|---|
-| `--bootstrap-token-path` | | File containing a bootstrap token. |
+| `--bootstrap-token-path` | | File containing a bootstrap token. A node that keeps this file can also recover on its own: when the control plane answers `401` to one of its authenticated calls (a control plane that lost its store, such as a new Cloud Run revision without a database), the node reads the token again and enrolls again at the next sync, without a restart; `agentmesh_node_reenrollments_total{result}` counts it. |
| `--bootstrap-token` | | The token as a value. Visible in process listings. The file form is preferred. |
| `--offline-access` | `false` | Request an OIDC refresh token, so that the node can re-enroll on its own when its credential expires. |
| `--allow-loopback` | `false` | Advertise and dial loopback and link-local addresses. For several nodes on one host. |