Skip to content

Agent Mesh

control plane client: test a fleet through the enrollment limiter

Anpoo Sivanadi1025890847e0
diff --git a/internal/controlplane/client/retry.go b/internal/controlplane/client/retry.go
index 2ae3904b..60f850b2 100644
--- a/internal/controlplane/client/retry.go
+++ b/internal/controlplane/client/retry.go
@@ -45,6 +45,10 @@ type Backoff struct {
// plane's limiter of ten a second. A larger fleet starts in waves.
var DefaultEnrollBackoff = Backoff{Initial: time.Second, Max: 15 * time.Second, Budget: 3 * time.Minute}
+// after is the wait between attempts. A test swaps it for a virtual clock so
+// minutes of backoff run in milliseconds.
+var after = time.After
+
// DoWithChallengeRetry is DoWithChallenge for a request the control plane
// may answer with 429 or 503. Such an answer is retried after Retry-After
// when the control plane names a wait, and after the backoff's own delay
@@ -80,7 +84,7 @@ func DoWithChallengeRetry(ctx context.Context, httpClient *http.Client, now func
select {
case <-ctx.Done():
return nil, ctx.Err()
- case <-time.After(wait):
+ case <-after(wait):
}
}
}
diff --git a/internal/controlplane/client/retry_fleet_test.go b/internal/controlplane/client/retry_fleet_test.go
new file mode 100644
index 00000000..5d766e7d
--- /dev/null
+++ b/internal/controlplane/client/retry_fleet_test.go
@@ -0,0 +1,218 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+ "container/heap"
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "sync"
+ "testing"
+ "time"
+
+ "golang.org/x/time/rate"
+
+ "github.com/google/agentmesh/internal/controlplane"
+)
+
+// fleetClock is a virtual clock shared by a fleet of callers. Time stands
+// still while any caller is working and jumps to the earliest pending wait
+// once every caller is either waiting or done, so the run is the same
+// sequence of events a real fleet would see, without the real waits.
+type fleetClock struct {
+ mu sync.Mutex
+ now time.Time
+ active int
+ waiters waitHeap
+}
+
+type fleetWaiter struct {
+ at time.Time
+ ch chan time.Time
+}
+
+// waitHeap orders pending waits by wake time.
+type waitHeap []fleetWaiter
+
+func (h waitHeap) Len() int { return len(h) }
+func (h waitHeap) Less(i, j int) bool { return h[i].at.Before(h[j].at) }
+func (h waitHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
+func (h *waitHeap) Push(x any) { *h = append(*h, x.(fleetWaiter)) }
+func (h *waitHeap) Pop() any {
+ old := *h
+ w := old[len(old)-1]
+ *h = old[:len(old)-1]
+ return w
+}
+
+func (c *fleetClock) Now() time.Time {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.now
+}
+
+// after stands in for time.After: the caller stops being active until the
+// clock reaches its wake time.
+func (c *fleetClock) after(d time.Duration) <-chan time.Time {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ w := fleetWaiter{at: c.now.Add(d), ch: make(chan time.Time, 1)}
+ heap.Push(&c.waiters, w)
+ c.idle()
+ return w.ch
+}
+
+// done marks a caller that returned.
+func (c *fleetClock) done() {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.idle()
+}
+
+func (c *fleetClock) idle() {
+ c.active--
+ if c.active > 0 || len(c.waiters) == 0 {
+ return
+ }
+ next := c.waiters[0].at
+ c.now = next
+ for len(c.waiters) > 0 && c.waiters[0].at.Equal(next) {
+ w := heap.Pop(&c.waiters).(fleetWaiter)
+ c.active++
+ w.ch <- next
+ }
+}
+
+// handlerTransport serves requests in process, so thousands of attempts
+// cost no sockets.
+type handlerTransport struct{ h http.Handler }
+
+func (t handlerTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+ rec := httptest.NewRecorder()
+ t.h.ServeHTTP(rec, r)
+ return rec.Result(), nil
+}
+
+// TestDoWithChallengeRetryFleet checks the sizing claim in the
+// DefaultEnrollBackoff comment: a fleet that starts at once behind the
+// control plane's enrollment limiter (EnrollRateLimit a second, burst
+// EnrollBurst, refusals answered with the same Retry-After as
+// tooManyRequests) is admitted within the budget when it is no larger than
+// burst + rate x budget, and past that every member left over gets the
+// limiter's last 429 back before the budget runs out. It also checks that
+// jitter spreads the fleet out after a shared Retry-After instead of
+// bringing it back in lockstep.
+func TestDoWithChallengeRetryFleet(t *testing.T) {
+ retryAfterSecs := max(1, controlplane.EnrollBurst/controlplane.EnrollRateLimit)
+ tests := []struct {
+ name string
+ members int
+ backoff Backoff
+ }{
+ {name: "1500 members within the default budget", members: 1500, backoff: DefaultEnrollBackoff},
+ {name: "fleet past the budget", members: 600, backoff: Backoff{Initial: time.Second, Max: 15 * time.Second, Budget: 30 * time.Second}},
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ start := time.Date(2026, 10, 13, 0, 0, 0, 0, time.UTC)
+ clock := &fleetClock{now: start, active: tc.members}
+ prev := after
+ after = clock.after
+ t.Cleanup(func() { after = prev })
+
+ limiter := rate.NewLimiter(rate.Limit(controlplane.EnrollRateLimit), controlplane.EnrollBurst)
+ var mu sync.Mutex
+ retries := map[time.Time]int{}
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ now := clock.Now()
+ if r.Header.Get("X-Attempt") != "1" {
+ mu.Lock()
+ retries[now]++
+ mu.Unlock()
+ }
+ if !limiter.AllowN(now, 1) {
+ w.Header().Set("Retry-After", strconv.Itoa(retryAfterSecs))
+ http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)
+ return
+ }
+ w.WriteHeader(http.StatusOK)
+ })
+ httpClient := &http.Client{Transport: handlerTransport{h: handler}}
+
+ type result struct {
+ status int
+ at time.Time
+ }
+ results := make([]result, tc.members)
+ var wg sync.WaitGroup
+ for i := range tc.members {
+ wg.Go(func() {
+ defer clock.done()
+ attempt := 0
+ resp, err := DoWithChallengeRetry(context.Background(), httpClient, clock.Now, tc.backoff, func(ts int64) (*http.Request, error) {
+ attempt++
+ req, err := http.NewRequest(http.MethodPost, "http://control-plane.test/enroll", nil)
+ if err == nil {
+ req.Header.Set("X-Attempt", strconv.Itoa(attempt))
+ }
+ return req, err
+ })
+ if err != nil {
+ t.Errorf("member %d: %v", i, err)
+ return
+ }
+ _ = resp.Body.Close()
+ results[i] = result{status: resp.StatusCode, at: clock.Now()}
+ })
+ }
+ wg.Wait()
+
+ deadline := start.Add(tc.backoff.Budget)
+ capacity := controlplane.EnrollBurst + int(float64(controlplane.EnrollRateLimit)*tc.backoff.Budget.Seconds())
+ admitted := 0
+ for i, r := range results {
+ if r.at.After(deadline) {
+ t.Errorf("member %d returned at %v, after the %v budget", i, r.at.Sub(start), tc.backoff.Budget)
+ }
+ switch r.status {
+ case http.StatusOK:
+ admitted++
+ case http.StatusTooManyRequests:
+ default:
+ t.Errorf("member %d got status %d, want 200 or 429", i, r.status)
+ }
+ }
+ if tc.members <= capacity && admitted != tc.members {
+ t.Errorf("admitted %d of %d members within %v, want all (capacity %d)", admitted, tc.members, tc.backoff.Budget, capacity)
+ }
+ if tc.members > capacity && (admitted == tc.members || admitted > capacity) {
+ t.Errorf("admitted %d of %d members, want between 1 and capacity %d with the rest refused", admitted, tc.members, capacity)
+ }
+
+ // Everyone refused at the start is told the same Retry-After; with
+ // jitter the retries land on many instants, not one.
+ busiest, total := 0, 0
+ for _, n := range retries {
+ busiest = max(busiest, n)
+ total += n
+ }
+ if total == 0 || busiest*10 > total {
+ t.Errorf("busiest retry instant took %d of %d retries, want the fleet spread out", busiest, total)
+ }
+ })
+ }
+}