Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
control plane client: test a fleet through the enrollment limiter
Anpoo Sivanadi1025890847e0
diff --git a/internal/controlplane/client/retry.go b/internal/controlplane/client/retry.goindex 2ae3904b..60f850b2 100644--- a/internal/controlplane/client/retry.go+++ b/internal/controlplane/client/retry.go@@ -45,6 +45,10 @@ type Backoff struct {// plane's limiter of ten a second. A larger fleet starts in waves.var DefaultEnrollBackoff = Backoff{Initial: time.Second, Max: 15 * time.Second, Budget: 3 * time.Minute}+// after is the wait between attempts. A test swaps it for a virtual clock so+// minutes of backoff run in milliseconds.+var after = time.After+// DoWithChallengeRetry is DoWithChallenge for a request the control plane// may answer with 429 or 503. Such an answer is retried after Retry-After// when the control plane names a wait, and after the backoff's own delay@@ -80,7 +84,7 @@ func DoWithChallengeRetry(ctx context.Context, httpClient *http.Client, now funcselect {case <-ctx.Done():return nil, ctx.Err()- case <-time.After(wait):+ case <-after(wait):}}}diff --git a/internal/controlplane/client/retry_fleet_test.go b/internal/controlplane/client/retry_fleet_test.gonew file mode 100644index 00000000..5d766e7d--- /dev/null+++ b/internal/controlplane/client/retry_fleet_test.go@@ -0,0 +1,218 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package client++import (+ "container/heap"+ "context"+ "net/http"+ "net/http/httptest"+ "strconv"+ "sync"+ "testing"+ "time"++ "golang.org/x/time/rate"++ "github.com/google/agentmesh/internal/controlplane"+)++// fleetClock is a virtual clock shared by a fleet of callers. Time stands+// still while any caller is working and jumps to the earliest pending wait+// once every caller is either waiting or done, so the run is the same+// sequence of events a real fleet would see, without the real waits.+type fleetClock struct {+ mu sync.Mutex+ now time.Time+ active int+ waiters waitHeap+}++type fleetWaiter struct {+ at time.Time+ ch chan time.Time+}++// waitHeap orders pending waits by wake time.+type waitHeap []fleetWaiter++func (h waitHeap) Len() int { return len(h) }+func (h waitHeap) Less(i, j int) bool { return h[i].at.Before(h[j].at) }+func (h waitHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }+func (h *waitHeap) Push(x any) { *h = append(*h, x.(fleetWaiter)) }+func (h *waitHeap) Pop() any {+ old := *h+ w := old[len(old)-1]+ *h = old[:len(old)-1]+ return w+}++func (c *fleetClock) Now() time.Time {+ c.mu.Lock()+ defer c.mu.Unlock()+ return c.now+}++// after stands in for time.After: the caller stops being active until the+// clock reaches its wake time.+func (c *fleetClock) after(d time.Duration) <-chan time.Time {+ c.mu.Lock()+ defer c.mu.Unlock()+ w := fleetWaiter{at: c.now.Add(d), ch: make(chan time.Time, 1)}+ heap.Push(&c.waiters, w)+ c.idle()+ return w.ch+}++// done marks a caller that returned.+func (c *fleetClock) done() {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.idle()+}++func (c *fleetClock) idle() {+ c.active--+ if c.active > 0 || len(c.waiters) == 0 {+ return+ }+ next := c.waiters[0].at+ c.now = next+ for len(c.waiters) > 0 && c.waiters[0].at.Equal(next) {+ w := heap.Pop(&c.waiters).(fleetWaiter)+ c.active+++ w.ch <- next+ }+}++// handlerTransport serves requests in process, so thousands of attempts+// cost no sockets.+type handlerTransport struct{ h http.Handler }++func (t handlerTransport) RoundTrip(r *http.Request) (*http.Response, error) {+ rec := httptest.NewRecorder()+ t.h.ServeHTTP(rec, r)+ return rec.Result(), nil+}++// TestDoWithChallengeRetryFleet checks the sizing claim in the+// DefaultEnrollBackoff comment: a fleet that starts at once behind the+// control plane's enrollment limiter (EnrollRateLimit a second, burst+// EnrollBurst, refusals answered with the same Retry-After as+// tooManyRequests) is admitted within the budget when it is no larger than+// burst + rate x budget, and past that every member left over gets the+// limiter's last 429 back before the budget runs out. It also checks that+// jitter spreads the fleet out after a shared Retry-After instead of+// bringing it back in lockstep.+func TestDoWithChallengeRetryFleet(t *testing.T) {+ retryAfterSecs := max(1, controlplane.EnrollBurst/controlplane.EnrollRateLimit)+ tests := []struct {+ name string+ members int+ backoff Backoff+ }{+ {name: "1500 members within the default budget", members: 1500, backoff: DefaultEnrollBackoff},+ {name: "fleet past the budget", members: 600, backoff: Backoff{Initial: time.Second, Max: 15 * time.Second, Budget: 30 * time.Second}},+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ start := time.Date(2026, 10, 13, 0, 0, 0, 0, time.UTC)+ clock := &fleetClock{now: start, active: tc.members}+ prev := after+ after = clock.after+ t.Cleanup(func() { after = prev })++ limiter := rate.NewLimiter(rate.Limit(controlplane.EnrollRateLimit), controlplane.EnrollBurst)+ var mu sync.Mutex+ retries := map[time.Time]int{}+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ now := clock.Now()+ if r.Header.Get("X-Attempt") != "1" {+ mu.Lock()+ retries[now]+++ mu.Unlock()+ }+ if !limiter.AllowN(now, 1) {+ w.Header().Set("Retry-After", strconv.Itoa(retryAfterSecs))+ http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)+ return+ }+ w.WriteHeader(http.StatusOK)+ })+ httpClient := &http.Client{Transport: handlerTransport{h: handler}}++ type result struct {+ status int+ at time.Time+ }+ results := make([]result, tc.members)+ var wg sync.WaitGroup+ for i := range tc.members {+ wg.Go(func() {+ defer clock.done()+ attempt := 0+ resp, err := DoWithChallengeRetry(context.Background(), httpClient, clock.Now, tc.backoff, func(ts int64) (*http.Request, error) {+ attempt+++ req, err := http.NewRequest(http.MethodPost, "http://control-plane.test/enroll", nil)+ if err == nil {+ req.Header.Set("X-Attempt", strconv.Itoa(attempt))+ }+ return req, err+ })+ if err != nil {+ t.Errorf("member %d: %v", i, err)+ return+ }+ _ = resp.Body.Close()+ results[i] = result{status: resp.StatusCode, at: clock.Now()}+ })+ }+ wg.Wait()++ deadline := start.Add(tc.backoff.Budget)+ capacity := controlplane.EnrollBurst + int(float64(controlplane.EnrollRateLimit)*tc.backoff.Budget.Seconds())+ admitted := 0+ for i, r := range results {+ if r.at.After(deadline) {+ t.Errorf("member %d returned at %v, after the %v budget", i, r.at.Sub(start), tc.backoff.Budget)+ }+ switch r.status {+ case http.StatusOK:+ admitted+++ case http.StatusTooManyRequests:+ default:+ t.Errorf("member %d got status %d, want 200 or 429", i, r.status)+ }+ }+ if tc.members <= capacity && admitted != tc.members {+ t.Errorf("admitted %d of %d members within %v, want all (capacity %d)", admitted, tc.members, tc.backoff.Budget, capacity)+ }+ if tc.members > capacity && (admitted == tc.members || admitted > capacity) {+ t.Errorf("admitted %d of %d members, want between 1 and capacity %d with the rest refused", admitted, tc.members, capacity)+ }++ // Everyone refused at the start is told the same Retry-After; with+ // jitter the retries land on many instants, not one.+ busiest, total := 0, 0+ for _, n := range retries {+ busiest = max(busiest, n)+ total += n+ }+ if total == 0 || busiest*10 > total {+ t.Errorf("busiest retry instant took %d of %d retries, want the fleet spread out", busiest, total)+ }+ })+ }+}