Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane/client: run the fleet retry test under synctest
Anpoo Sivanadi6bdce6c69a3a
diff --git a/internal/controlplane/client/retry.go b/internal/controlplane/client/retry.goindex 60f850b2..2ae3904b 100644--- a/internal/controlplane/client/retry.go+++ b/internal/controlplane/client/retry.go@@ -45,10 +45,6 @@ type Backoff struct {// plane's limiter of ten a second. A larger fleet starts in waves.var DefaultEnrollBackoff = Backoff{Initial: time.Second, Max: 15 * time.Second, Budget: 3 * time.Minute}-// after is the wait between attempts. A test swaps it for a virtual clock so-// minutes of backoff run in milliseconds.-var after = time.After-// DoWithChallengeRetry is DoWithChallenge for a request the control plane// may answer with 429 or 503. Such an answer is retried after Retry-After// when the control plane names a wait, and after the backoff's own delay@@ -84,7 +80,7 @@ func DoWithChallengeRetry(ctx context.Context, httpClient *http.Client, now funcselect {case <-ctx.Done():return nil, ctx.Err()- case <-after(wait):+ case <-time.After(wait):}}}diff --git a/internal/controlplane/client/retry_fleet_test.go b/internal/controlplane/client/retry_fleet_test.goindex 5d766e7d..cc372e74 100644--- a/internal/controlplane/client/retry_fleet_test.go+++ b/internal/controlplane/client/retry_fleet_test.go@@ -15,13 +15,13 @@package clientimport (- "container/heap""context""net/http""net/http/httptest""strconv""sync""testing"+ "testing/synctest""time""golang.org/x/time/rate"@@ -29,74 +29,6 @@ import ("github.com/google/agentmesh/internal/controlplane")-// fleetClock is a virtual clock shared by a fleet of callers. Time stands-// still while any caller is working and jumps to the earliest pending wait-// once every caller is either waiting or done, so the run is the same-// sequence of events a real fleet would see, without the real waits.-type fleetClock struct {- mu sync.Mutex- now time.Time- active int- waiters waitHeap-}--type fleetWaiter struct {- at time.Time- ch chan time.Time-}--// waitHeap orders pending waits by wake time.-type waitHeap []fleetWaiter--func (h waitHeap) Len() int { return len(h) }-func (h waitHeap) Less(i, j int) bool { return h[i].at.Before(h[j].at) }-func (h waitHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }-func (h *waitHeap) Push(x any) { *h = append(*h, x.(fleetWaiter)) }-func (h *waitHeap) Pop() any {- old := *h- w := old[len(old)-1]- *h = old[:len(old)-1]- return w-}--func (c *fleetClock) Now() time.Time {- c.mu.Lock()- defer c.mu.Unlock()- return c.now-}--// after stands in for time.After: the caller stops being active until the-// clock reaches its wake time.-func (c *fleetClock) after(d time.Duration) <-chan time.Time {- c.mu.Lock()- defer c.mu.Unlock()- w := fleetWaiter{at: c.now.Add(d), ch: make(chan time.Time, 1)}- heap.Push(&c.waiters, w)- c.idle()- return w.ch-}--// done marks a caller that returned.-func (c *fleetClock) done() {- c.mu.Lock()- defer c.mu.Unlock()- c.idle()-}--func (c *fleetClock) idle() {- c.active--- if c.active > 0 || len(c.waiters) == 0 {- return- }- next := c.waiters[0].at- c.now = next- for len(c.waiters) > 0 && c.waiters[0].at.Equal(next) {- w := heap.Pop(&c.waiters).(fleetWaiter)- c.active++- w.ch <- next- }-}-// handlerTransport serves requests in process, so thousands of attempts// cost no sockets.type handlerTransport struct{ h http.Handler }@@ -115,7 +47,8 @@ func (t handlerTransport) RoundTrip(r *http.Request) (*http.Response, error) {// burst + rate x budget, and past that every member left over gets the// limiter's last 429 back before the budget runs out. It also checks that// jitter spreads the fleet out after a shared Retry-After instead of-// bringing it back in lockstep.+// bringing it back in lockstep. synctest runs the fleet on a fake clock, so+// minutes of backoff take milliseconds.func TestDoWithChallengeRetryFleet(t *testing.T) {retryAfterSecs := max(1, controlplane.EnrollBurst/controlplane.EnrollRateLimit)tests := []struct {@@ -128,91 +61,88 @@ func TestDoWithChallengeRetryFleet(t *testing.T) {}for _, tc := range tests {t.Run(tc.name, func(t *testing.T) {- start := time.Date(2026, 10, 13, 0, 0, 0, 0, time.UTC)- clock := &fleetClock{now: start, active: tc.members}- prev := after- after = clock.after- t.Cleanup(func() { after = prev })+ synctest.Test(t, func(t *testing.T) {+ start := time.Now()++ limiter := rate.NewLimiter(rate.Limit(controlplane.EnrollRateLimit), controlplane.EnrollBurst)+ var mu sync.Mutex+ retries := map[time.Time]int{}+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ now := time.Now()+ if r.Header.Get("X-Attempt") != "1" {+ mu.Lock()+ retries[now]+++ mu.Unlock()+ }+ if !limiter.AllowN(now, 1) {+ w.Header().Set("Retry-After", strconv.Itoa(retryAfterSecs))+ http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)+ return+ }+ w.WriteHeader(http.StatusOK)+ })+ httpClient := &http.Client{Transport: handlerTransport{h: handler}}- limiter := rate.NewLimiter(rate.Limit(controlplane.EnrollRateLimit), controlplane.EnrollBurst)- var mu sync.Mutex- retries := map[time.Time]int{}- handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- now := clock.Now()- if r.Header.Get("X-Attempt") != "1" {- mu.Lock()- retries[now]++- mu.Unlock()+ type result struct {+ status int+ at time.Time}- if !limiter.AllowN(now, 1) {- w.Header().Set("Retry-After", strconv.Itoa(retryAfterSecs))- http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests)- return- }- w.WriteHeader(http.StatusOK)- })- httpClient := &http.Client{Transport: handlerTransport{h: handler}}-- type result struct {- status int- at time.Time- }- results := make([]result, tc.members)- var wg sync.WaitGroup- for i := range tc.members {- wg.Go(func() {- defer clock.done()- attempt := 0- resp, err := DoWithChallengeRetry(context.Background(), httpClient, clock.Now, tc.backoff, func(ts int64) (*http.Request, error) {- attempt++- req, err := http.NewRequest(http.MethodPost, "http://control-plane.test/enroll", nil)- if err == nil {- req.Header.Set("X-Attempt", strconv.Itoa(attempt))+ results := make([]result, tc.members)+ var wg sync.WaitGroup+ for i := range tc.members {+ wg.Go(func() {+ attempt := 0+ resp, err := DoWithChallengeRetry(context.Background(), httpClient, time.Now, tc.backoff, func(ts int64) (*http.Request, error) {+ attempt+++ req, err := http.NewRequest(http.MethodPost, "http://control-plane.test/enroll", nil)+ if err == nil {+ req.Header.Set("X-Attempt", strconv.Itoa(attempt))+ }+ return req, err+ })+ if err != nil {+ t.Errorf("member %d: %v", i, err)+ return}- return req, err+ _ = resp.Body.Close()+ results[i] = result{status: resp.StatusCode, at: time.Now()}})- if err != nil {- t.Errorf("member %d: %v", i, err)- return+ }+ wg.Wait()++ deadline := start.Add(tc.backoff.Budget)+ capacity := controlplane.EnrollBurst + int(float64(controlplane.EnrollRateLimit)*tc.backoff.Budget.Seconds())+ admitted := 0+ for i, r := range results {+ if r.at.After(deadline) {+ t.Errorf("member %d returned at %v, after the %v budget", i, r.at.Sub(start), tc.backoff.Budget)+ }+ switch r.status {+ case http.StatusOK:+ admitted+++ case http.StatusTooManyRequests:+ default:+ t.Errorf("member %d got status %d, want 200 or 429", i, r.status)}- _ = resp.Body.Close()- results[i] = result{status: resp.StatusCode, at: clock.Now()}- })- }- wg.Wait()-- deadline := start.Add(tc.backoff.Budget)- capacity := controlplane.EnrollBurst + int(float64(controlplane.EnrollRateLimit)*tc.backoff.Budget.Seconds())- admitted := 0- for i, r := range results {- if r.at.After(deadline) {- t.Errorf("member %d returned at %v, after the %v budget", i, r.at.Sub(start), tc.backoff.Budget)}- switch r.status {- case http.StatusOK:- admitted++- case http.StatusTooManyRequests:- default:- t.Errorf("member %d got status %d, want 200 or 429", i, r.status)+ if tc.members <= capacity && admitted != tc.members {+ t.Errorf("admitted %d of %d members within %v, want all (capacity %d)", admitted, tc.members, tc.backoff.Budget, capacity)+ }+ if tc.members > capacity && (admitted == tc.members || admitted > capacity) {+ t.Errorf("admitted %d of %d members, want between 1 and capacity %d with the rest refused", admitted, tc.members, capacity)}- }- if tc.members <= capacity && admitted != tc.members {- t.Errorf("admitted %d of %d members within %v, want all (capacity %d)", admitted, tc.members, tc.backoff.Budget, capacity)- }- if tc.members > capacity && (admitted == tc.members || admitted > capacity) {- t.Errorf("admitted %d of %d members, want between 1 and capacity %d with the rest refused", admitted, tc.members, capacity)- }- // Everyone refused at the start is told the same Retry-After; with- // jitter the retries land on many instants, not one.- busiest, total := 0, 0- for _, n := range retries {- busiest = max(busiest, n)- total += n- }- if total == 0 || busiest*10 > total {- t.Errorf("busiest retry instant took %d of %d retries, want the fleet spread out", busiest, total)- }+ // Everyone refused at the start is told the same Retry-After; with+ // jitter the retries land on many instants, not one.+ busiest, total := 0, 0+ for _, n := range retries {+ busiest = max(busiest, n)+ total += n+ }+ if total == 0 || busiest*10 > total {+ t.Errorf("busiest retry instant took %d of %d retries, want the fleet spread out", busiest, total)+ }+ })})}}