Skip to content

Agent Mesh

node, control plane: retry an enrollment the limiter turned away

Antonio Ojeafd633fa452ea
diff --git a/internal/controlplane/client/retry.go b/internal/controlplane/client/retry.go
new file mode 100644
index 00000000..e03da32d
--- /dev/null
+++ b/internal/controlplane/client/retry.go
@@ -0,0 +1,108 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+ "context"
+ "io"
+ "math/rand/v2"
+ "net/http"
+ "strconv"
+ "time"
+)
+
+// Backoff is how a caller waits for a control plane that is busy. A 429 or
+// 503 is the control plane saying "not now": its enrollment limiter admits
+// a fixed number of members per second for the whole mesh, so a fleet that
+// joins at once is told to come back, and a member that gave up at the
+// first such answer never joined.
+type Backoff struct {
+ // Initial is the first wait when the answer carries no Retry-After;
+ // each further wait doubles, up to Max.
+ Initial, Max time.Duration
+ // Attempts is how many answers are retried before the last one is
+ // returned to the caller as it came.
+ Attempts int
+}
+
+// DefaultEnrollBackoff waits about a minute in all before giving up, which
+// covers a fleet of a few hundred members joining through a limiter of ten
+// per second.
+var DefaultEnrollBackoff = Backoff{Initial: time.Second, Max: 15 * time.Second, Attempts: 8}
+
+// DoWithChallengeRetry is DoWithChallenge for a request the control plane
+// may answer with 429 or 503. Such an answer is retried after Retry-After
+// when the control plane names a wait, and after the backoff's own delay
+// when it does not, with jitter so a fleet told "not now" together does
+// not come back together. Any other answer, and the last retried one, is
+// returned to the caller.
+func DoWithChallengeRetry(ctx context.Context, httpClient *http.Client, now func() time.Time, b Backoff, build func(ts int64) (*http.Request, error)) (*http.Response, error) {
+ if now == nil {
+ now = time.Now
+ }
+ delay := b.Initial
+ for attempt := 0; ; attempt++ {
+ resp, err := DoWithChallenge(httpClient, now, build)
+ if err != nil {
+ return nil, err
+ }
+ if !retryable(resp.StatusCode) || attempt >= b.Attempts {
+ return resp, nil
+ }
+ wait, named := retryAfter(resp, now)
+ if !named {
+ wait = delay
+ delay = min(2*delay, b.Max)
+ }
+ // Up to half the wait again, at random.
+ wait += time.Duration(rand.Int64N(int64(wait)/2 + 1))
+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, MaxBodyBytes))
+ _ = resp.Body.Close()
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(wait):
+ }
+ }
+}
+
+func retryable(status int) bool {
+ return status == http.StatusTooManyRequests || status == http.StatusServiceUnavailable
+}
+
+// retryAfter reads a Retry-After header in either of its forms. The wait is
+// capped at a minute: a header is advice, and a control plane's clock or
+// typo must not park a member for an hour.
+func retryAfter(resp *http.Response, now func() time.Time) (time.Duration, bool) {
+ h := resp.Header.Get("Retry-After")
+ if h == "" {
+ return 0, false
+ }
+ const cap = time.Minute
+ if secs, err := strconv.Atoi(h); err == nil {
+ if secs < 0 {
+ return 0, false
+ }
+ return min(time.Duration(secs)*time.Second, cap), true
+ }
+ if at, err := http.ParseTime(h); err == nil {
+ wait := at.Sub(now())
+ if wait < 0 {
+ wait = 0
+ }
+ return min(wait, cap), true
+ }
+ return 0, false
+}
diff --git a/internal/controlplane/client/retry_test.go b/internal/controlplane/client/retry_test.go
new file mode 100644
index 00000000..11369915
--- /dev/null
+++ b/internal/controlplane/client/retry_test.go
@@ -0,0 +1,181 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// TestDoWithChallengeRetry covers a control plane that is busy: 429 and 503
+// are retried, Retry-After sets the wait when present, other answers are
+// returned at once, and a limiter that never relents hands its last answer
+// back after the configured attempts.
+func TestDoWithChallengeRetry(t *testing.T) {
+ fast := Backoff{Initial: time.Millisecond, Max: 5 * time.Millisecond, Attempts: 4}
+ build := func(url string) func(ts int64) (*http.Request, error) {
+ return func(ts int64) (*http.Request, error) {
+ return http.NewRequest(http.MethodPost, url, nil)
+ }
+ }
+
+ t.Run("busy then admitted", func(t *testing.T) {
+ var calls atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch calls.Add(1) {
+ case 1:
+ http.Error(w, "busy", http.StatusTooManyRequests)
+ case 2:
+ http.Error(w, "busy", http.StatusServiceUnavailable)
+ default:
+ w.WriteHeader(http.StatusOK)
+ }
+ }))
+ defer srv.Close()
+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusOK || calls.Load() != 3 {
+ t.Fatalf("status %d after %d calls, want 200 after 3", resp.StatusCode, calls.Load())
+ }
+ })
+
+ t.Run("retry-after is honoured", func(t *testing.T) {
+ var calls atomic.Int32
+ var gap atomic.Int64
+ var first time.Time
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch calls.Add(1) {
+ case 1:
+ first = time.Now()
+ w.Header().Set("Retry-After", "1")
+ http.Error(w, "busy", http.StatusTooManyRequests)
+ default:
+ gap.Store(int64(time.Since(first)))
+ w.WriteHeader(http.StatusOK)
+ }
+ }))
+ defer srv.Close()
+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ // At least the named second, at most that plus half again of jitter.
+ if got := time.Duration(gap.Load()); got < time.Second || got > 1600*time.Millisecond {
+ t.Fatalf("waited %s after Retry-After: 1, want 1s to 1.5s", got)
+ }
+ })
+
+ t.Run("retry-after as a date, with the default clock", func(t *testing.T) {
+ var calls atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if calls.Add(1) == 1 {
+ w.Header().Set("Retry-After", time.Now().Add(200*time.Millisecond).UTC().Format(http.TimeFormat))
+ http.Error(w, "busy", http.StatusTooManyRequests)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer srv.Close()
+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusOK || calls.Load() != 2 {
+ t.Fatalf("status %d after %d calls, want 200 after 2", resp.StatusCode, calls.Load())
+ }
+ })
+
+ t.Run("other answers return at once", func(t *testing.T) {
+ var calls atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls.Add(1)
+ http.Error(w, "no", http.StatusForbidden)
+ }))
+ defer srv.Close()
+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusForbidden || calls.Load() != 1 {
+ t.Fatalf("status %d after %d calls, want 403 after 1", resp.StatusCode, calls.Load())
+ }
+ })
+
+ t.Run("gives up after the attempts", func(t *testing.T) {
+ var calls atomic.Int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls.Add(1)
+ http.Error(w, "busy", http.StatusTooManyRequests)
+ }))
+ defer srv.Close()
+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusTooManyRequests || int(calls.Load()) != fast.Attempts+1 {
+ t.Fatalf("status %d after %d calls, want 429 after %d", resp.StatusCode, calls.Load(), fast.Attempts+1)
+ }
+ })
+
+ t.Run("cancelled while waiting", func(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Retry-After", "30")
+ http.Error(w, "busy", http.StatusTooManyRequests)
+ }))
+ defer srv.Close()
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancel()
+ if _, err := DoWithChallengeRetry(ctx, srv.Client(), nil, fast, build(srv.URL)); err != context.DeadlineExceeded {
+ t.Fatalf("err = %v, want context deadline", err)
+ }
+ })
+}
+
+func TestRetryAfter(t *testing.T) {
+ now := func() time.Time { return time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) }
+ for _, tc := range []struct {
+ header string
+ want time.Duration
+ named bool
+ }{
+ {"", 0, false},
+ {"3", 3 * time.Second, true},
+ {"-1", 0, false},
+ {"7200", time.Minute, true},
+ {now().Add(5 * time.Second).Format(http.TimeFormat), 5 * time.Second, true},
+ {now().Add(-5 * time.Second).Format(http.TimeFormat), 0, true},
+ {"soon", 0, false},
+ } {
+ resp := &http.Response{Header: http.Header{}}
+ if tc.header != "" {
+ resp.Header.Set("Retry-After", tc.header)
+ }
+ got, named := retryAfter(resp, now)
+ if got != tc.want || named != tc.named {
+ t.Errorf("Retry-After %q: got %s/%v, want %s/%v", tc.header, got, named, tc.want, tc.named)
+ }
+ }
+}
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 4bcfc232..a7e99567 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -683,7 +683,7 @@ func (s *Server) HandleRegister(w http.ResponseWriter, r *http.Request) {
}
if !s.limiter.Allow() {
- http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
+ tooManyRequests(w)
return
}
@@ -1875,7 +1875,7 @@ func (s *Server) HandleEnroll(w http.ResponseWriter, r *http.Request) {
}
if !s.limiter.Allow() {
- http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
+ tooManyRequests(w)
return
}
@@ -2172,6 +2172,15 @@ func writeChallengeError(w http.ResponseWriter, msg string) {
http.Error(w, msg, http.StatusUnauthorized)
}
+// tooManyRequests answers a request the enrollment limiter refused. The
+// limiter admits EnrollRateLimit a second for the whole mesh, so a fleet
+// that joins at once is told when to come back; the wait spreads the fleet
+// over the seconds the limiter needs for it. Clients that retry read it.
+func tooManyRequests(w http.ResponseWriter) {
+ w.Header().Set("Retry-After", strconv.Itoa(max(1, EnrollBurst/EnrollRateLimit)))
+ http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
+}
+
// HandleEnrollStatus HTTP GET `/enroll/status`
//
// The approved response carries the enrollee's Biscuit, so polling requires
@@ -2188,7 +2197,7 @@ func (s *Server) HandleEnrollStatus(w http.ResponseWriter, r *http.Request) {
}
if !s.limiter.Allow() {
- http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
+ tooManyRequests(w)
return
}
diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.go
index 00d6d407..df421881 100644
--- a/internal/controlplane/server_test.go
+++ b/internal/controlplane/server_test.go
@@ -1540,6 +1540,11 @@ func TestEnrollStatusRateLimited(t *testing.T) {
}
_ = resp.Body.Close()
if resp.StatusCode == http.StatusTooManyRequests {
+ // The limiter names the wait, so a fleet turned away comes
+ // back spread over the seconds the limiter needs for it.
+ if got, err := strconv.Atoi(resp.Header.Get("Retry-After")); err != nil || got < 1 {
+ t.Fatalf("429 carries Retry-After %q, want a positive number of seconds", resp.Header.Get("Retry-After"))
+ }
limited = true
break
}
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index 1a0084ae..9240bfcc 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -115,7 +115,7 @@ func (n *AgentMeshNode) enrollHTTP(ctx context.Context, controlPlaneURL, jwt str
logger.Infof("Enrolling via HTTP at %s", url)
client := controlPlaneHTTPClient(30 * time.Second)
- resp, err := cpclient.DoWithChallenge(client, time.Now, func(ts int64) (*http.Request, error) {
+ resp, err := cpclient.DoWithChallengeRetry(ctx, client, time.Now, cpclient.DefaultEnrollBackoff, func(ts int64) (*http.Request, error) {
sig, err := privKey.Sign(api.RegisterChallenge(peerID.String(), ts))
if err != nil {
return nil, fmt.Errorf("failed to sign registration challenge: %w", err)
@@ -265,7 +265,7 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL str
logger.Infof("Enrolling via Bootstrap token at %s", enrollURL)
client := controlPlaneHTTPClient(30 * time.Second)
- resp, err := cpclient.DoWithChallenge(client, time.Now, func(enrollTS int64) (*http.Request, error) {
+ resp, err := cpclient.DoWithChallengeRetry(ctx, client, time.Now, cpclient.DefaultEnrollBackoff, func(enrollTS int64) (*http.Request, error) {
enrollSig, err := n.config.PrivKey.Sign(api.EnrollChallenge(n.Host.ID().String(), enrollTS))
if err != nil {
return nil, fmt.Errorf("failed to sign enrollment challenge: %w", err)
diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.md
index 001e08bd..40aa3dad 100644
--- a/site/content/docs/reference/control-plane.md
+++ b/site/content/docs/reference/control-plane.md
@@ -73,6 +73,14 @@ instant. This proves that the caller holds the key behind the peer ID it
names. Every instant in a response (`expire_time` and the like) is a
`google.protobuf.Timestamp`, an RFC 3339 string in JSON.
+Enrollment is admitted at 10 requests a second for the whole mesh, with a
+burst of 20. A request over that is answered `429` with a `Retry-After`
+header. `agentmesh-node` waits for it and sends the request again, with
+jitter, for about a minute before it reports the failure, so a fleet of a
+few hundred members that starts at once joins over its first minute instead
+of losing the members the limiter turned away. A client of your own should
+do the same.
+
| Route | Body | Purpose |
|---|---|---|
| `POST /register` | `EnrollRequest` (OIDC token, public key, requested role, labels) | OIDC enrollment. Returns `EnrollResponse`: the credential, the control plane public key, router addresses and expiry. |