Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node, control plane: retry an enrollment the limiter turned away
Antonio Ojeafd633fa452ea
diff --git a/internal/controlplane/client/retry.go b/internal/controlplane/client/retry.gonew file mode 100644index 00000000..e03da32d--- /dev/null+++ b/internal/controlplane/client/retry.go@@ -0,0 +1,108 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package client++import (+ "context"+ "io"+ "math/rand/v2"+ "net/http"+ "strconv"+ "time"+)++// Backoff is how a caller waits for a control plane that is busy. A 429 or+// 503 is the control plane saying "not now": its enrollment limiter admits+// a fixed number of members per second for the whole mesh, so a fleet that+// joins at once is told to come back, and a member that gave up at the+// first such answer never joined.+type Backoff struct {+ // Initial is the first wait when the answer carries no Retry-After;+ // each further wait doubles, up to Max.+ Initial, Max time.Duration+ // Attempts is how many answers are retried before the last one is+ // returned to the caller as it came.+ Attempts int+}++// DefaultEnrollBackoff waits about a minute in all before giving up, which+// covers a fleet of a few hundred members joining through a limiter of ten+// per second.+var DefaultEnrollBackoff = Backoff{Initial: time.Second, Max: 15 * time.Second, Attempts: 8}++// DoWithChallengeRetry is DoWithChallenge for a request the control plane+// may answer with 429 or 503. Such an answer is retried after Retry-After+// when the control plane names a wait, and after the backoff's own delay+// when it does not, with jitter so a fleet told "not now" together does+// not come back together. Any other answer, and the last retried one, is+// returned to the caller.+func DoWithChallengeRetry(ctx context.Context, httpClient *http.Client, now func() time.Time, b Backoff, build func(ts int64) (*http.Request, error)) (*http.Response, error) {+ if now == nil {+ now = time.Now+ }+ delay := b.Initial+ for attempt := 0; ; attempt++ {+ resp, err := DoWithChallenge(httpClient, now, build)+ if err != nil {+ return nil, err+ }+ if !retryable(resp.StatusCode) || attempt >= b.Attempts {+ return resp, nil+ }+ wait, named := retryAfter(resp, now)+ if !named {+ wait = delay+ delay = min(2*delay, b.Max)+ }+ // Up to half the wait again, at random.+ wait += time.Duration(rand.Int64N(int64(wait)/2 + 1))+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, MaxBodyBytes))+ _ = resp.Body.Close()+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(wait):+ }+ }+}++func retryable(status int) bool {+ return status == http.StatusTooManyRequests || status == http.StatusServiceUnavailable+}++// retryAfter reads a Retry-After header in either of its forms. The wait is+// capped at a minute: a header is advice, and a control plane's clock or+// typo must not park a member for an hour.+func retryAfter(resp *http.Response, now func() time.Time) (time.Duration, bool) {+ h := resp.Header.Get("Retry-After")+ if h == "" {+ return 0, false+ }+ const cap = time.Minute+ if secs, err := strconv.Atoi(h); err == nil {+ if secs < 0 {+ return 0, false+ }+ return min(time.Duration(secs)*time.Second, cap), true+ }+ if at, err := http.ParseTime(h); err == nil {+ wait := at.Sub(now())+ if wait < 0 {+ wait = 0+ }+ return min(wait, cap), true+ }+ return 0, false+}diff --git a/internal/controlplane/client/retry_test.go b/internal/controlplane/client/retry_test.gonew file mode 100644index 00000000..11369915--- /dev/null+++ b/internal/controlplane/client/retry_test.go@@ -0,0 +1,181 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package client++import (+ "context"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"+ "time"+)++// TestDoWithChallengeRetry covers a control plane that is busy: 429 and 503+// are retried, Retry-After sets the wait when present, other answers are+// returned at once, and a limiter that never relents hands its last answer+// back after the configured attempts.+func TestDoWithChallengeRetry(t *testing.T) {+ fast := Backoff{Initial: time.Millisecond, Max: 5 * time.Millisecond, Attempts: 4}+ build := func(url string) func(ts int64) (*http.Request, error) {+ return func(ts int64) (*http.Request, error) {+ return http.NewRequest(http.MethodPost, url, nil)+ }+ }++ t.Run("busy then admitted", func(t *testing.T) {+ var calls atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch calls.Add(1) {+ case 1:+ http.Error(w, "busy", http.StatusTooManyRequests)+ case 2:+ http.Error(w, "busy", http.StatusServiceUnavailable)+ default:+ w.WriteHeader(http.StatusOK)+ }+ }))+ defer srv.Close()+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))+ if err != nil {+ t.Fatal(err)+ }+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusOK || calls.Load() != 3 {+ t.Fatalf("status %d after %d calls, want 200 after 3", resp.StatusCode, calls.Load())+ }+ })++ t.Run("retry-after is honoured", func(t *testing.T) {+ var calls atomic.Int32+ var gap atomic.Int64+ var first time.Time+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch calls.Add(1) {+ case 1:+ first = time.Now()+ w.Header().Set("Retry-After", "1")+ http.Error(w, "busy", http.StatusTooManyRequests)+ default:+ gap.Store(int64(time.Since(first)))+ w.WriteHeader(http.StatusOK)+ }+ }))+ defer srv.Close()+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))+ if err != nil {+ t.Fatal(err)+ }+ _ = resp.Body.Close()+ // At least the named second, at most that plus half again of jitter.+ if got := time.Duration(gap.Load()); got < time.Second || got > 1600*time.Millisecond {+ t.Fatalf("waited %s after Retry-After: 1, want 1s to 1.5s", got)+ }+ })++ t.Run("retry-after as a date, with the default clock", func(t *testing.T) {+ var calls atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if calls.Add(1) == 1 {+ w.Header().Set("Retry-After", time.Now().Add(200*time.Millisecond).UTC().Format(http.TimeFormat))+ http.Error(w, "busy", http.StatusTooManyRequests)+ return+ }+ w.WriteHeader(http.StatusOK)+ }))+ defer srv.Close()+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))+ if err != nil {+ t.Fatal(err)+ }+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusOK || calls.Load() != 2 {+ t.Fatalf("status %d after %d calls, want 200 after 2", resp.StatusCode, calls.Load())+ }+ })++ t.Run("other answers return at once", func(t *testing.T) {+ var calls atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ calls.Add(1)+ http.Error(w, "no", http.StatusForbidden)+ }))+ defer srv.Close()+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))+ if err != nil {+ t.Fatal(err)+ }+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusForbidden || calls.Load() != 1 {+ t.Fatalf("status %d after %d calls, want 403 after 1", resp.StatusCode, calls.Load())+ }+ })++ t.Run("gives up after the attempts", func(t *testing.T) {+ var calls atomic.Int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ calls.Add(1)+ http.Error(w, "busy", http.StatusTooManyRequests)+ }))+ defer srv.Close()+ resp, err := DoWithChallengeRetry(context.Background(), srv.Client(), nil, fast, build(srv.URL))+ if err != nil {+ t.Fatal(err)+ }+ _ = resp.Body.Close()+ if resp.StatusCode != http.StatusTooManyRequests || int(calls.Load()) != fast.Attempts+1 {+ t.Fatalf("status %d after %d calls, want 429 after %d", resp.StatusCode, calls.Load(), fast.Attempts+1)+ }+ })++ t.Run("cancelled while waiting", func(t *testing.T) {+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Retry-After", "30")+ http.Error(w, "busy", http.StatusTooManyRequests)+ }))+ defer srv.Close()+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)+ defer cancel()+ if _, err := DoWithChallengeRetry(ctx, srv.Client(), nil, fast, build(srv.URL)); err != context.DeadlineExceeded {+ t.Fatalf("err = %v, want context deadline", err)+ }+ })+}++func TestRetryAfter(t *testing.T) {+ now := func() time.Time { return time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) }+ for _, tc := range []struct {+ header string+ want time.Duration+ named bool+ }{+ {"", 0, false},+ {"3", 3 * time.Second, true},+ {"-1", 0, false},+ {"7200", time.Minute, true},+ {now().Add(5 * time.Second).Format(http.TimeFormat), 5 * time.Second, true},+ {now().Add(-5 * time.Second).Format(http.TimeFormat), 0, true},+ {"soon", 0, false},+ } {+ resp := &http.Response{Header: http.Header{}}+ if tc.header != "" {+ resp.Header.Set("Retry-After", tc.header)+ }+ got, named := retryAfter(resp, now)+ if got != tc.want || named != tc.named {+ t.Errorf("Retry-After %q: got %s/%v, want %s/%v", tc.header, got, named, tc.want, tc.named)+ }+ }+}diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex 4bcfc232..a7e99567 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -683,7 +683,7 @@ func (s *Server) HandleRegister(w http.ResponseWriter, r *http.Request) {}if !s.limiter.Allow() {- http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)+ tooManyRequests(w)return}@@ -1875,7 +1875,7 @@ func (s *Server) HandleEnroll(w http.ResponseWriter, r *http.Request) {}if !s.limiter.Allow() {- http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)+ tooManyRequests(w)return}@@ -2172,6 +2172,15 @@ func writeChallengeError(w http.ResponseWriter, msg string) {http.Error(w, msg, http.StatusUnauthorized)}+// tooManyRequests answers a request the enrollment limiter refused. The+// limiter admits EnrollRateLimit a second for the whole mesh, so a fleet+// that joins at once is told when to come back; the wait spreads the fleet+// over the seconds the limiter needs for it. Clients that retry read it.+func tooManyRequests(w http.ResponseWriter) {+ w.Header().Set("Retry-After", strconv.Itoa(max(1, EnrollBurst/EnrollRateLimit)))+ http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)+}+// HandleEnrollStatus HTTP GET `/enroll/status`//// The approved response carries the enrollee's Biscuit, so polling requires@@ -2188,7 +2197,7 @@ func (s *Server) HandleEnrollStatus(w http.ResponseWriter, r *http.Request) {}if !s.limiter.Allow() {- http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)+ tooManyRequests(w)return}diff --git a/internal/controlplane/server_test.go b/internal/controlplane/server_test.goindex 00d6d407..df421881 100644--- a/internal/controlplane/server_test.go+++ b/internal/controlplane/server_test.go@@ -1540,6 +1540,11 @@ func TestEnrollStatusRateLimited(t *testing.T) {}_ = resp.Body.Close()if resp.StatusCode == http.StatusTooManyRequests {+ // The limiter names the wait, so a fleet turned away comes+ // back spread over the seconds the limiter needs for it.+ if got, err := strconv.Atoi(resp.Header.Get("Retry-After")); err != nil || got < 1 {+ t.Fatalf("429 carries Retry-After %q, want a positive number of seconds", resp.Header.Get("Retry-After"))+ }limited = truebreak}diff --git a/internal/node/enroll.go b/internal/node/enroll.goindex 1a0084ae..9240bfcc 100644--- a/internal/node/enroll.go+++ b/internal/node/enroll.go@@ -115,7 +115,7 @@ func (n *AgentMeshNode) enrollHTTP(ctx context.Context, controlPlaneURL, jwt strlogger.Infof("Enrolling via HTTP at %s", url)client := controlPlaneHTTPClient(30 * time.Second)- resp, err := cpclient.DoWithChallenge(client, time.Now, func(ts int64) (*http.Request, error) {+ resp, err := cpclient.DoWithChallengeRetry(ctx, client, time.Now, cpclient.DefaultEnrollBackoff, func(ts int64) (*http.Request, error) {sig, err := privKey.Sign(api.RegisterChallenge(peerID.String(), ts))if err != nil {return nil, fmt.Errorf("failed to sign registration challenge: %w", err)@@ -265,7 +265,7 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL strlogger.Infof("Enrolling via Bootstrap token at %s", enrollURL)client := controlPlaneHTTPClient(30 * time.Second)- resp, err := cpclient.DoWithChallenge(client, time.Now, func(enrollTS int64) (*http.Request, error) {+ resp, err := cpclient.DoWithChallengeRetry(ctx, client, time.Now, cpclient.DefaultEnrollBackoff, func(enrollTS int64) (*http.Request, error) {enrollSig, err := n.config.PrivKey.Sign(api.EnrollChallenge(n.Host.ID().String(), enrollTS))if err != nil {return nil, fmt.Errorf("failed to sign enrollment challenge: %w", err)diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.mdindex 001e08bd..40aa3dad 100644--- a/site/content/docs/reference/control-plane.md+++ b/site/content/docs/reference/control-plane.md@@ -73,6 +73,14 @@ instant. This proves that the caller holds the key behind the peer ID itnames. Every instant in a response (`expire_time` and the like) is a`google.protobuf.Timestamp`, an RFC 3339 string in JSON.+Enrollment is admitted at 10 requests a second for the whole mesh, with a+burst of 20. A request over that is answered `429` with a `Retry-After`+header. `agentmesh-node` waits for it and sends the request again, with+jitter, for about a minute before it reports the failure, so a fleet of a+few hundred members that starts at once joins over its first minute instead+of losing the members the limiter turned away. A client of your own should+do the same.+| Route | Body | Purpose ||---|---|---|| `POST /register` | `EnrollRequest` (OIDC token, public key, requested role, labels) | OIDC enrollment. Returns `EnrollResponse`: the credential, the control plane public key, router addresses and expiry. |