Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: the TestMaxRPM tests hold the limiter's clock, so a busy machine can't turn them red (#1791)

yetone2c3c903974b6
diff --git a/internal/gateway/rpm.go b/internal/gateway/rpm.go
index dd414ba48..4bf0eb43a 100644
--- a/internal/gateway/rpm.go
+++ b/internal/gateway/rpm.go
@@ -58,6 +58,17 @@ func rpmSpan() time.Duration { return rpmWindow + rpmSlack }
// the provider's QueueWait isn't shorter; tests shorten it.
var rpmLongest = 2 * time.Minute
+// rpmNow is the clock the minute is read on, and rpmTimer the timer a
+// request waits for room on; tests hold both, so what they check doesn't
+// rest on how fast a loaded machine gets to it (#1791).
+var (
+ rpmNow = time.Now
+ rpmTimer = func(d time.Duration) (<-chan time.Time, func() bool) {
+ t := time.NewTimer(d)
+ return t.C, t.Stop
+ }
+)
+
// rpms are the times of the requests of each key or account with a MaxRPM.
type rpms struct {
mu sync.Mutex
@@ -150,7 +161,7 @@ func (l *rpms) wait(ctx context.Context, who string, limit int, longest time.Dur
if limit <= 0 {
return 0, nil
}
- now := time.Now()
+ now := rpmNow()
at, err := l.reserve(who, limit, longest, now)
if err != nil {
return 0, err
@@ -160,14 +171,14 @@ func (l *rpms) wait(ctx context.Context, who string, limit int, longest time.Dur
return 0, nil
}
defer notePhase(ctx, "rpm")()
- t := time.NewTimer(d)
- defer t.Stop()
+ c, stop := rpmTimer(d)
+ defer stop()
select {
- case <-t.C:
+ case <-c:
return d, nil
case <-ctx.Done():
l.giveBack(who, at)
- return time.Since(now), ctx.Err()
+ return rpmNow().Sub(now), ctx.Err()
}
}
@@ -182,7 +193,7 @@ func (l *rpms) free(who string, limit int) bool {
if l.m == nil {
return true
}
- return len(l.prune(who, time.Now())) < limit
+ return len(l.prune(who, rpmNow())) < limit
}
// used is how many requests of who's are counted in its minute now, to go
@@ -193,7 +204,7 @@ func (l *rpms) used(who string) int {
if l.m == nil {
return 0
}
- return len(l.prune(who, time.Now()))
+ return len(l.prune(who, rpmNow()))
}
// rpmTicket, on a request's context, counts each request made with it
diff --git a/internal/gateway/rpm_test.go b/internal/gateway/rpm_test.go
index ee4134d06..8310b76f6 100644
--- a/internal/gateway/rpm_test.go
+++ b/internal/gateway/rpm_test.go
@@ -3,9 +3,11 @@ package gateway
import (
"context"
"encoding/json"
+ "fmt"
"io"
"net/http"
"net/http/httptest"
+ "slices"
"strings"
"sync"
"testing"
@@ -26,7 +28,11 @@ type minuteVendor struct {
// when its connection has to be dialed (and, to a real vendor, its TLS
// handshake made) while the later ones reuse it
late time.Duration
- n int
+ // clock, when set, is MaxRPM's held clock (holdMinute): the vendor
+ // notes each request's time on it, and a late first moves it on by
+ // late rather than sleeping
+ clock *minuteClock
+ n int
}
func (v *minuteVendor) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -34,11 +40,19 @@ func (v *minuteVendor) ServeHTTP(w http.ResponseWriter, r *http.Request) {
v.mu.Lock()
v.n++
if v.n == 1 && v.late > 0 {
- v.mu.Unlock()
- time.Sleep(v.late)
- v.mu.Lock()
+ if v.clock != nil {
+ v.clock.advance(v.late)
+ } else {
+ v.mu.Unlock()
+ time.Sleep(v.late)
+ v.mu.Lock()
+ }
+ }
+ if v.clock != nil {
+ v.at = append(v.at, v.clock.Now())
+ } else {
+ v.at = append(v.at, time.Now())
}
- v.at = append(v.at, time.Now())
v.path = append(v.path, r.URL.Path)
failing := v.fail > 0 && strings.HasSuffix(r.URL.Path, "/chat/completions")
if failing {
@@ -82,6 +96,187 @@ func slackMinute(t *testing.T, w, longest time.Duration) {
rpmSlack = w / 6
}
+// minuteClock is MaxRPM's clock and timer held still: it moves only when the
+// test moves it. A test of the minute on the wall clock went red on a busy
+// machine whenever the machine was slower than the checks allowed: a second
+// request that came late in a 1.5s window was told Retry-After 1, or found
+// the window passed (#1791). On the held clock, what the limiter decides
+// rests on the minute's arithmetic alone, and a wait for room is a state
+// the test sees (waiting) rather than a time it measures.
+type minuteClock struct {
+ mu sync.Mutex
+ now time.Time
+ waits []*minuteWait
+}
+
+type minuteWait struct {
+ at time.Time
+ c chan time.Time
+}
+
+// holdMinute holds MaxRPM's clock for the test, before its gateway starts.
+// A minuteVendor given the clock lets every wait go before the gateway
+// closes (rpmGateway), so a request a failed test left waiting ends.
+func holdMinute(t *testing.T) *minuteClock {
+ t.Helper()
+ c := &minuteClock{now: time.Now()}
+ on, ot := rpmNow, rpmTimer
+ rpmNow, rpmTimer = c.Now, c.timer
+ t.Cleanup(func() { rpmNow, rpmTimer = on, ot })
+ return c
+}
+
+func (c *minuteClock) Now() time.Time {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.now
+}
+
+func (c *minuteClock) timer(d time.Duration) (<-chan time.Time, func() bool) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ w := &minuteWait{at: c.now.Add(d), c: make(chan time.Time, 1)}
+ if !w.at.After(c.now) {
+ w.c <- c.now
+ return w.c, func() bool { return false }
+ }
+ c.waits = append(c.waits, w)
+ return w.c, func() bool {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ i := slices.Index(c.waits, w)
+ if i < 0 {
+ return false
+ }
+ c.waits = slices.Delete(c.waits, i, i+1)
+ return true
+ }
+}
+
+// advance moves the clock on by d, letting go the waits it reaches.
+func (c *minuteClock) advance(d time.Duration) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.now = c.now.Add(d)
+ c.waits = slices.DeleteFunc(c.waits, func(w *minuteWait) bool {
+ if w.at.After(c.now) {
+ return false
+ }
+ w.c <- c.now
+ return true
+ })
+}
+
+// waiting is when each request waiting for room in the minute goes.
+func (c *minuteClock) waiting() []time.Time {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ var at []time.Time
+ for _, w := range c.waits {
+ at = append(at, w.at)
+ }
+ return at
+}
+
+// release lets every wait go.
+func (c *minuteClock) release() {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ for _, w := range c.waits {
+ w.c <- w.at
+ }
+ c.waits = nil
+}
+
+// rpmReply is a request's answer, or the error that ended it.
+type rpmReply struct {
+ code int
+ header http.Header
+ body string
+ err error
+}
+
+// askRPM sends body to the gateway's path and answers on the channel once
+// the request ends.
+func askRPM(ctx context.Context, gw, path, body string) <-chan rpmReply {
+ out := make(chan rpmReply, 1)
+ go func() {
+ req, _ := http.NewRequestWithContext(ctx, "POST", gw+path, strings.NewReader(body))
+ req.Header.Set("Content-Type", "application/json")
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ out <- rpmReply{err: err}
+ return
+ }
+ b, _ := io.ReadAll(res.Body)
+ res.Body.Close()
+ out <- rpmReply{code: res.StatusCode, header: res.Header, body: string(b)}
+ }()
+ return out
+}
+
+const rlPath, rlBody = "/v1/chat/completions", `{"model":"rl/m","messages":[{"role":"user","content":"hi"}]}`
+
+// rpmBound bounds how long a test waits for a request on a busy machine to
+// get where it is going: never a check of how fast it was, only so a test
+// gone wrong ends.
+const rpmBound = 2 * time.Minute
+
+// atOnce is r's answer, failing the test if its request starts a wait for
+// room in the minute instead: on the held clock such a wait never ends on
+// its own, so the request either answers without one or is seen waiting.
+func (c *minuteClock) atOnce(t *testing.T, what string, r <-chan rpmReply) rpmReply {
+ t.Helper()
+ for end := time.Now().Add(rpmBound); time.Now().Before(end); {
+ select {
+ case got := <-r:
+ return got
+ case <-time.After(5 * time.Millisecond):
+ }
+ if w := c.waiting(); len(w) > 0 {
+ c.release()
+ t.Fatalf("%s waited for room in the minute, to go %s after now", what, w[0].Sub(c.Now()))
+ }
+ }
+ t.Fatalf("%s didn't end in %s", what, rpmBound)
+ return rpmReply{}
+}
+
+// waitsUntil is when the one request waiting for room in the minute goes,
+// once it is waiting.
+func (c *minuteClock) waitsUntil(t *testing.T, what string) time.Time {
+ t.Helper()
+ var w []time.Time
+ rpmEventually(t, what+" waiting for room in the minute", func() bool { w = c.waiting(); return len(w) > 0 })
+ if len(w) != 1 {
+ t.Fatalf("%d waiting for room in the minute, want %s alone", len(w), what)
+ }
+ return w[0]
+}
+
+// rpmAnswered is r's answer, once its request ends.
+func rpmAnswered(t *testing.T, what string, r <-chan rpmReply) rpmReply {
+ t.Helper()
+ select {
+ case got := <-r:
+ return got
+ case <-time.After(rpmBound):
+ t.Fatalf("%s didn't end in %s", what, rpmBound)
+ }
+ return rpmReply{}
+}
+
+// rpmEventually waits for cond, as within does, with room for a busy machine.
+func rpmEventually(t *testing.T, what string, cond func() bool) {
+ t.Helper()
+ for end := time.Now().Add(rpmBound); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
+ if cond() {
+ return
+ }
+ }
+ t.Fatalf("timed out waiting for %s", what)
+}
+
// rpmGateway is a gateway as the real server serves it (lanGuard in
// front), with a provider "rl" of one key whose MaxRPM is limit.
func rpmGateway(t *testing.T, v *minuteVendor, limit int) (*Server, string) {
@@ -95,45 +290,49 @@ func rpmGateway(t *testing.T, v *minuteVendor, limit int) (*Server, string) {
s := New()
gw := httptest.NewServer(lanGuard(s.Handler()))
t.Cleanup(gw.Close)
- return s, gw.URL
-}
-
-func chatOnce(ctx context.Context, gw string) (*http.Response, string, error) {
- req, _ := http.NewRequestWithContext(ctx, "POST", gw+"/v1/chat/completions", strings.NewReader(`{"model":"rl/m","messages":[{"role":"user","content":"hi"}]}`))
- req.Header.Set("Content-Type", "application/json")
- res, err := http.DefaultClient.Do(req)
- if err != nil {
- return nil, "", err
+ if v.clock != nil {
+ // before the gateway closes, which waits for its requests
+ t.Cleanup(v.clock.release)
}
- b, _ := io.ReadAll(res.Body)
- res.Body.Close()
- return res, string(b), nil
+ return s, gw.URL
}
// coeo91 on Discord: OpenRouter's free models take 20 requests a minute,
// which a limit at once of 1 didn't keep to. With a MaxRPM of 2, through
// the real server, two requests go at once and the third waits until the
-// first is a minute old, then is answered.
+// first is a minute (and the slack) old, then is answered; the vendor,
+// whose first request came late by its dial, still sees the third a whole
+// window after the first.
func TestMaxRPMWaitsForRoomInTheMinute(t *testing.T) {
window := 1500 * time.Millisecond
slackMinute(t, window, time.Minute)
// the first dials the connection the others reuse
- v := &minuteVendor{late: 150 * time.Millisecond}
+ v := &minuteVendor{late: 150 * time.Millisecond, clock: holdMinute(t)}
_, gw := rpmGateway(t, v, 2)
- start := time.Now()
- for i := range 3 {
- res, body, err := chatOnce(context.Background(), gw)
- if err != nil || res.StatusCode != 200 || !strings.Contains(body, `"ok"`) {
- t.Fatalf("request %d: %v %v %s", i+1, err, res, body)
+ clk := v.clock
+ t0 := clk.Now()
+ for i := range 2 {
+ what := fmt.Sprintf("request %d, with room in the minute,", i+1)
+ if r := clk.atOnce(t, what, askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 || !strings.Contains(r.body, `"ok"`) {
+ t.Fatalf("request %d: %v %d %s", i+1, r.err, r.code, r.body)
}
}
+ third := askRPM(context.Background(), gw, rlPath, rlBody)
+ until := clk.waitsUntil(t, "the third")
+ if want := t0.Add(window + rpmSlack); !until.Equal(want) {
+ t.Fatalf("the third waits to go %s after the first, want the window and slack, %s", until.Sub(t0), want.Sub(t0))
+ }
+ if n := len(v.times()); n != 2 {
+ t.Fatalf("the vendor saw %d while the third waits, want 2", n)
+ }
+ clk.advance(until.Sub(clk.Now()))
+ if r := rpmAnswered(t, "the third", third); r.err != nil || r.code != 200 || !strings.Contains(r.body, `"ok"`) {
+ t.Fatalf("request 3: %v %d %s", r.err, r.code, r.body)
+ }
at := v.times()
if len(at) != 3 {
t.Fatalf("the vendor saw %d requests, want 3", len(at))
}
- if d := at[1].Sub(start); d > window/2 {
- t.Fatalf("the second waited %s with room in the minute", d)
- }
if d := at[2].Sub(at[0]); d < window {
t.Fatalf("the vendor saw the third %s after the first, inside the %s window", d, window)
}
@@ -146,21 +345,20 @@ func TestMaxRPMWaitsForRoomInTheMinute(t *testing.T) {
func TestMaxRPMTooLongIs429WithRetryAfter(t *testing.T) {
window := 1500 * time.Millisecond
shortMinute(t, window, 200*time.Millisecond)
- v := &minuteVendor{}
+ v := &minuteVendor{clock: holdMinute(t)}
_, gw := rpmGateway(t, v, 1)
- if res, body, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {
- t.Fatalf("first: %v %v %s", err, res, body)
- }
- began := time.Now()
- res, body, err := chatOnce(context.Background(), gw)
- if err != nil {
- t.Fatal(err)
+ clk := v.clock
+ if r := clk.atOnce(t, "the first", askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 {
+ t.Fatalf("first: %v %d %s", r.err, r.code, r.body)
}
- if res.StatusCode != 429 || res.Header.Get("Retry-After") != "2" || !strings.Contains(body, "1 requests a minute is its limit") {
- t.Fatalf("second = %d Retry-After %q %s, want a 429 saying the limit, Retry-After 2", res.StatusCode, res.Header.Get("Retry-After"), body)
+ // the clock held where the first went, the minute has room again in
+ // the whole window, 1.5s: Retry-After 2
+ r := clk.atOnce(t, "the second, to be turned away,", askRPM(context.Background(), gw, rlPath, rlBody))
+ if r.err != nil {
+ t.Fatal(r.err)
}
- if d := time.Since(began); d > window/2 {
- t.Fatalf("turned away after %s, not at once", d)
+ if r.code != 429 || r.header.Get("Retry-After") != "2" || !strings.Contains(r.body, "1 requests a minute is its limit") {
+ t.Fatalf("second = %d Retry-After %q %s, want a 429 saying the limit, Retry-After 2", r.code, r.header.Get("Retry-After"), r.body)
}
if n := len(v.times()); n != 1 {
t.Fatalf("the vendor saw %d, want 1", n)
@@ -168,9 +366,9 @@ func TestMaxRPMTooLongIs429WithRetryAfter(t *testing.T) {
if r := restingNow(t); len(r) != 0 {
t.Fatalf("%v rest for a limit of the user's", r)
}
- time.Sleep(window)
- if res, body, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {
- t.Fatalf("after the minute: %v %v %s", err, res, body)
+ clk.advance(window)
+ if r := clk.atOnce(t, "the next, after the minute,", askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 {
+ t.Fatalf("after the minute: %v %d %s", r.err, r.code, r.body)
}
}
@@ -178,25 +376,29 @@ func TestMaxRPMTooLongIs429WithRetryAfter(t *testing.T) {
// nothing is sent, and its place in the minute is given back.
func TestMaxRPMAgentGoneWhileWaiting(t *testing.T) {
shortMinute(t, 3*time.Second, time.Minute)
- v := &minuteVendor{}
+ v := &minuteVendor{clock: holdMinute(t)}
s, gw := rpmGateway(t, v, 1)
- if res, _, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {
- t.Fatalf("first: %v %v", err, res)
+ clk := v.clock
+ if r := clk.atOnce(t, "the first", askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 {
+ t.Fatalf("first: %v %d", r.err, r.code)
}
who := meterWho(provider.Provider{ID: "rl", Key: "k"})
ctx, cancel := context.WithCancel(context.Background())
- done := make(chan error, 1)
- go func() {
- _, _, err := chatOnce(ctx, gw)
- done <- err
- }()
- within(t, "the second waiting", func() bool { return s.rpms.used(who) == 2 })
+ second := askRPM(ctx, gw, rlPath, rlBody)
+ clk.waitsUntil(t, "the second")
+ if n := s.rpms.used(who); n != 2 {
+ t.Fatalf("%d counted with the second waiting, want 2", n)
+ }
cancel()
- if err := <-done; err == nil {
+ if r := rpmAnswered(t, "the canceled request", second); r.err == nil {
t.Fatal("the canceled request answered")
}
- within(t, "its place given back", func() bool { return s.rpms.used(who) == 1 })
- time.Sleep(100 * time.Millisecond)
+ rpmEventually(t, "its place given back", func() bool { return s.rpms.used(who) == 1 })
+ // its wait ended with its place given back, so nothing is left to send
+ // it once the minute has room
+ if w := clk.waiting(); len(w) != 0 {
+ t.Fatalf("%d still waiting for room after the agent went", len(w))
+ }
if n := len(v.times()); n != 1 {
t.Fatalf("the vendor saw %d, want 1", n)
}
@@ -209,47 +411,41 @@ func TestMaxRPMAgentGoneWhileWaiting(t *testing.T) {
func TestMaxRPMCountsRetriesAndCounts(t *testing.T) {
window := 1500 * time.Millisecond
slackMinute(t, window, time.Minute)
- v := &minuteVendor{fail: 1, late: 150 * time.Millisecond}
+ v := &minuteVendor{fail: 1, late: 150 * time.Millisecond, clock: holdMinute(t)}
_, gw := rpmGateway(t, v, 3)
- if res, body, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {
- t.Fatalf("first: %v %v %s", err, res, body)
+ clk := v.clock
+ t0 := clk.Now()
+ if r := clk.atOnce(t, "the first", askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 {
+ t.Fatalf("first: %v %d %s", r.err, r.code, r.body)
}
if n := len(v.times()); n != 2 {
t.Fatalf("the vendor saw %d, want the 503 and its retry", n)
}
- req, _ := http.NewRequest("POST", gw+"/v1/messages/count_tokens", strings.NewReader(`{"model":"rl/m","messages":[{"role":"user","content":"hi"}]}`))
- req.Header.Set("Content-Type", "application/json")
- res, err := http.DefaultClient.Do(req)
- if err != nil {
- t.Fatal(err)
- }
+ r := clk.atOnce(t, "the count, with room in the minute,", askRPM(context.Background(), gw, "/v1/messages/count_tokens", rlBody))
var count struct {
InputTokens int `json:"input_tokens"`
}
- json.NewDecoder(res.Body).Decode(&count)
- res.Body.Close()
- if count.InputTokens != 7 {
- t.Fatalf("count = %d, want the vendor's 7", count.InputTokens)
+ json.Unmarshal([]byte(r.body), &count)
+ if r.err != nil || count.InputTokens != 7 {
+ t.Fatalf("count = %d (%v %s), want the vendor's 7", count.InputTokens, r.err, r.body)
}
// with no room in the minute, a count doesn't wait: the estimate
// answers, and the vendor isn't asked
- req, _ = http.NewRequest("POST", gw+"/v1/messages/count_tokens", strings.NewReader(`{"model":"rl/m","messages":[{"role":"user","content":"hi"}]}`))
- req.Header.Set("Content-Type", "application/json")
- began := time.Now()
- res, err = http.DefaultClient.Do(req)
- if err != nil {
- t.Fatal(err)
- }
- io.Copy(io.Discard, res.Body)
- res.Body.Close()
- if res.StatusCode != 200 || time.Since(began) > window/2 {
- t.Fatalf("a count with no room: %d after %s", res.StatusCode, time.Since(began))
+ r = clk.atOnce(t, "a count with no room", askRPM(context.Background(), gw, "/v1/messages/count_tokens", rlBody))
+ if r.err != nil || r.code != 200 {
+ t.Fatalf("a count with no room: %v %d %s", r.err, r.code, r.body)
}
if n := len(v.times()); n != 3 {
t.Fatalf("the vendor saw %d, want the count estimated", n)
}
- if res, body, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {
- t.Fatalf("last: %v %v %s", err, res, body)
+ last := askRPM(context.Background(), gw, rlPath, rlBody)
+ until := clk.waitsUntil(t, "the last")
+ if want := t0.Add(window + rpmSlack); !until.Equal(want) {
+ t.Fatalf("the last waits to go %s after the first, want the window and slack, %s: the retry or the count wasn't counted", until.Sub(t0), want.Sub(t0))
+ }
+ clk.advance(until.Sub(clk.Now()))
+ if r := rpmAnswered(t, "the last", last); r.err != nil || r.code != 200 {
+ t.Fatalf("last: %v %d %s", r.err, r.code, r.body)
}
at := v.times()
if len(at) != 4 {