Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: the TestMaxRPM tests hold the limiter's clock, so a busy machine can't turn them red (#1791)
yetone2c3c903974b6
diff --git a/internal/gateway/rpm.go b/internal/gateway/rpm.goindex dd414ba48..4bf0eb43a 100644--- a/internal/gateway/rpm.go+++ b/internal/gateway/rpm.go@@ -58,6 +58,17 @@ func rpmSpan() time.Duration { return rpmWindow + rpmSlack }// the provider's QueueWait isn't shorter; tests shorten it.var rpmLongest = 2 * time.Minute+// rpmNow is the clock the minute is read on, and rpmTimer the timer a+// request waits for room on; tests hold both, so what they check doesn't+// rest on how fast a loaded machine gets to it (#1791).+var (+ rpmNow = time.Now+ rpmTimer = func(d time.Duration) (<-chan time.Time, func() bool) {+ t := time.NewTimer(d)+ return t.C, t.Stop+ }+)+// rpms are the times of the requests of each key or account with a MaxRPM.type rpms struct {mu sync.Mutex@@ -150,7 +161,7 @@ func (l *rpms) wait(ctx context.Context, who string, limit int, longest time.Durif limit <= 0 {return 0, nil}- now := time.Now()+ now := rpmNow()at, err := l.reserve(who, limit, longest, now)if err != nil {return 0, err@@ -160,14 +171,14 @@ func (l *rpms) wait(ctx context.Context, who string, limit int, longest time.Durreturn 0, nil}defer notePhase(ctx, "rpm")()- t := time.NewTimer(d)- defer t.Stop()+ c, stop := rpmTimer(d)+ defer stop()select {- case <-t.C:+ case <-c:return d, nilcase <-ctx.Done():l.giveBack(who, at)- return time.Since(now), ctx.Err()+ return rpmNow().Sub(now), ctx.Err()}}@@ -182,7 +193,7 @@ func (l *rpms) free(who string, limit int) bool {if l.m == nil {return true}- return len(l.prune(who, time.Now())) < limit+ return len(l.prune(who, rpmNow())) < limit}// used is how many requests of who's are counted in its minute now, to go@@ -193,7 +204,7 @@ func (l *rpms) used(who string) int {if l.m == nil {return 0}- return len(l.prune(who, time.Now()))+ return len(l.prune(who, rpmNow()))}// rpmTicket, on a request's context, counts each request made with itdiff --git a/internal/gateway/rpm_test.go b/internal/gateway/rpm_test.goindex ee4134d06..8310b76f6 100644--- a/internal/gateway/rpm_test.go+++ b/internal/gateway/rpm_test.go@@ -3,9 +3,11 @@ package gatewayimport ("context""encoding/json"+ "fmt""io""net/http""net/http/httptest"+ "slices""strings""sync""testing"@@ -26,7 +28,11 @@ type minuteVendor struct {// when its connection has to be dialed (and, to a real vendor, its TLS// handshake made) while the later ones reuse itlate time.Duration- n int+ // clock, when set, is MaxRPM's held clock (holdMinute): the vendor+ // notes each request's time on it, and a late first moves it on by+ // late rather than sleeping+ clock *minuteClock+ n int}func (v *minuteVendor) ServeHTTP(w http.ResponseWriter, r *http.Request) {@@ -34,11 +40,19 @@ func (v *minuteVendor) ServeHTTP(w http.ResponseWriter, r *http.Request) {v.mu.Lock()v.n++if v.n == 1 && v.late > 0 {- v.mu.Unlock()- time.Sleep(v.late)- v.mu.Lock()+ if v.clock != nil {+ v.clock.advance(v.late)+ } else {+ v.mu.Unlock()+ time.Sleep(v.late)+ v.mu.Lock()+ }+ }+ if v.clock != nil {+ v.at = append(v.at, v.clock.Now())+ } else {+ v.at = append(v.at, time.Now())}- v.at = append(v.at, time.Now())v.path = append(v.path, r.URL.Path)failing := v.fail > 0 && strings.HasSuffix(r.URL.Path, "/chat/completions")if failing {@@ -82,6 +96,187 @@ func slackMinute(t *testing.T, w, longest time.Duration) {rpmSlack = w / 6}+// minuteClock is MaxRPM's clock and timer held still: it moves only when the+// test moves it. A test of the minute on the wall clock went red on a busy+// machine whenever the machine was slower than the checks allowed: a second+// request that came late in a 1.5s window was told Retry-After 1, or found+// the window passed (#1791). On the held clock, what the limiter decides+// rests on the minute's arithmetic alone, and a wait for room is a state+// the test sees (waiting) rather than a time it measures.+type minuteClock struct {+ mu sync.Mutex+ now time.Time+ waits []*minuteWait+}++type minuteWait struct {+ at time.Time+ c chan time.Time+}++// holdMinute holds MaxRPM's clock for the test, before its gateway starts.+// A minuteVendor given the clock lets every wait go before the gateway+// closes (rpmGateway), so a request a failed test left waiting ends.+func holdMinute(t *testing.T) *minuteClock {+ t.Helper()+ c := &minuteClock{now: time.Now()}+ on, ot := rpmNow, rpmTimer+ rpmNow, rpmTimer = c.Now, c.timer+ t.Cleanup(func() { rpmNow, rpmTimer = on, ot })+ return c+}++func (c *minuteClock) Now() time.Time {+ c.mu.Lock()+ defer c.mu.Unlock()+ return c.now+}++func (c *minuteClock) timer(d time.Duration) (<-chan time.Time, func() bool) {+ c.mu.Lock()+ defer c.mu.Unlock()+ w := &minuteWait{at: c.now.Add(d), c: make(chan time.Time, 1)}+ if !w.at.After(c.now) {+ w.c <- c.now+ return w.c, func() bool { return false }+ }+ c.waits = append(c.waits, w)+ return w.c, func() bool {+ c.mu.Lock()+ defer c.mu.Unlock()+ i := slices.Index(c.waits, w)+ if i < 0 {+ return false+ }+ c.waits = slices.Delete(c.waits, i, i+1)+ return true+ }+}++// advance moves the clock on by d, letting go the waits it reaches.+func (c *minuteClock) advance(d time.Duration) {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.now = c.now.Add(d)+ c.waits = slices.DeleteFunc(c.waits, func(w *minuteWait) bool {+ if w.at.After(c.now) {+ return false+ }+ w.c <- c.now+ return true+ })+}++// waiting is when each request waiting for room in the minute goes.+func (c *minuteClock) waiting() []time.Time {+ c.mu.Lock()+ defer c.mu.Unlock()+ var at []time.Time+ for _, w := range c.waits {+ at = append(at, w.at)+ }+ return at+}++// release lets every wait go.+func (c *minuteClock) release() {+ c.mu.Lock()+ defer c.mu.Unlock()+ for _, w := range c.waits {+ w.c <- w.at+ }+ c.waits = nil+}++// rpmReply is a request's answer, or the error that ended it.+type rpmReply struct {+ code int+ header http.Header+ body string+ err error+}++// askRPM sends body to the gateway's path and answers on the channel once+// the request ends.+func askRPM(ctx context.Context, gw, path, body string) <-chan rpmReply {+ out := make(chan rpmReply, 1)+ go func() {+ req, _ := http.NewRequestWithContext(ctx, "POST", gw+path, strings.NewReader(body))+ req.Header.Set("Content-Type", "application/json")+ res, err := http.DefaultClient.Do(req)+ if err != nil {+ out <- rpmReply{err: err}+ return+ }+ b, _ := io.ReadAll(res.Body)+ res.Body.Close()+ out <- rpmReply{code: res.StatusCode, header: res.Header, body: string(b)}+ }()+ return out+}++const rlPath, rlBody = "/v1/chat/completions", `{"model":"rl/m","messages":[{"role":"user","content":"hi"}]}`++// rpmBound bounds how long a test waits for a request on a busy machine to+// get where it is going: never a check of how fast it was, only so a test+// gone wrong ends.+const rpmBound = 2 * time.Minute++// atOnce is r's answer, failing the test if its request starts a wait for+// room in the minute instead: on the held clock such a wait never ends on+// its own, so the request either answers without one or is seen waiting.+func (c *minuteClock) atOnce(t *testing.T, what string, r <-chan rpmReply) rpmReply {+ t.Helper()+ for end := time.Now().Add(rpmBound); time.Now().Before(end); {+ select {+ case got := <-r:+ return got+ case <-time.After(5 * time.Millisecond):+ }+ if w := c.waiting(); len(w) > 0 {+ c.release()+ t.Fatalf("%s waited for room in the minute, to go %s after now", what, w[0].Sub(c.Now()))+ }+ }+ t.Fatalf("%s didn't end in %s", what, rpmBound)+ return rpmReply{}+}++// waitsUntil is when the one request waiting for room in the minute goes,+// once it is waiting.+func (c *minuteClock) waitsUntil(t *testing.T, what string) time.Time {+ t.Helper()+ var w []time.Time+ rpmEventually(t, what+" waiting for room in the minute", func() bool { w = c.waiting(); return len(w) > 0 })+ if len(w) != 1 {+ t.Fatalf("%d waiting for room in the minute, want %s alone", len(w), what)+ }+ return w[0]+}++// rpmAnswered is r's answer, once its request ends.+func rpmAnswered(t *testing.T, what string, r <-chan rpmReply) rpmReply {+ t.Helper()+ select {+ case got := <-r:+ return got+ case <-time.After(rpmBound):+ t.Fatalf("%s didn't end in %s", what, rpmBound)+ }+ return rpmReply{}+}++// rpmEventually waits for cond, as within does, with room for a busy machine.+func rpmEventually(t *testing.T, what string, cond func() bool) {+ t.Helper()+ for end := time.Now().Add(rpmBound); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {+ if cond() {+ return+ }+ }+ t.Fatalf("timed out waiting for %s", what)+}+// rpmGateway is a gateway as the real server serves it (lanGuard in// front), with a provider "rl" of one key whose MaxRPM is limit.func rpmGateway(t *testing.T, v *minuteVendor, limit int) (*Server, string) {@@ -95,45 +290,49 @@ func rpmGateway(t *testing.T, v *minuteVendor, limit int) (*Server, string) {s := New()gw := httptest.NewServer(lanGuard(s.Handler()))t.Cleanup(gw.Close)- return s, gw.URL-}--func chatOnce(ctx context.Context, gw string) (*http.Response, string, error) {- req, _ := http.NewRequestWithContext(ctx, "POST", gw+"/v1/chat/completions", strings.NewReader(`{"model":"rl/m","messages":[{"role":"user","content":"hi"}]}`))- req.Header.Set("Content-Type", "application/json")- res, err := http.DefaultClient.Do(req)- if err != nil {- return nil, "", err+ if v.clock != nil {+ // before the gateway closes, which waits for its requests+ t.Cleanup(v.clock.release)}- b, _ := io.ReadAll(res.Body)- res.Body.Close()- return res, string(b), nil+ return s, gw.URL}// coeo91 on Discord: OpenRouter's free models take 20 requests a minute,// which a limit at once of 1 didn't keep to. With a MaxRPM of 2, through// the real server, two requests go at once and the third waits until the-// first is a minute old, then is answered.+// first is a minute (and the slack) old, then is answered; the vendor,+// whose first request came late by its dial, still sees the third a whole+// window after the first.func TestMaxRPMWaitsForRoomInTheMinute(t *testing.T) {window := 1500 * time.MillisecondslackMinute(t, window, time.Minute)// the first dials the connection the others reuse- v := &minuteVendor{late: 150 * time.Millisecond}+ v := &minuteVendor{late: 150 * time.Millisecond, clock: holdMinute(t)}_, gw := rpmGateway(t, v, 2)- start := time.Now()- for i := range 3 {- res, body, err := chatOnce(context.Background(), gw)- if err != nil || res.StatusCode != 200 || !strings.Contains(body, `"ok"`) {- t.Fatalf("request %d: %v %v %s", i+1, err, res, body)+ clk := v.clock+ t0 := clk.Now()+ for i := range 2 {+ what := fmt.Sprintf("request %d, with room in the minute,", i+1)+ if r := clk.atOnce(t, what, askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 || !strings.Contains(r.body, `"ok"`) {+ t.Fatalf("request %d: %v %d %s", i+1, r.err, r.code, r.body)}}+ third := askRPM(context.Background(), gw, rlPath, rlBody)+ until := clk.waitsUntil(t, "the third")+ if want := t0.Add(window + rpmSlack); !until.Equal(want) {+ t.Fatalf("the third waits to go %s after the first, want the window and slack, %s", until.Sub(t0), want.Sub(t0))+ }+ if n := len(v.times()); n != 2 {+ t.Fatalf("the vendor saw %d while the third waits, want 2", n)+ }+ clk.advance(until.Sub(clk.Now()))+ if r := rpmAnswered(t, "the third", third); r.err != nil || r.code != 200 || !strings.Contains(r.body, `"ok"`) {+ t.Fatalf("request 3: %v %d %s", r.err, r.code, r.body)+ }at := v.times()if len(at) != 3 {t.Fatalf("the vendor saw %d requests, want 3", len(at))}- if d := at[1].Sub(start); d > window/2 {- t.Fatalf("the second waited %s with room in the minute", d)- }if d := at[2].Sub(at[0]); d < window {t.Fatalf("the vendor saw the third %s after the first, inside the %s window", d, window)}@@ -146,21 +345,20 @@ func TestMaxRPMWaitsForRoomInTheMinute(t *testing.T) {func TestMaxRPMTooLongIs429WithRetryAfter(t *testing.T) {window := 1500 * time.MillisecondshortMinute(t, window, 200*time.Millisecond)- v := &minuteVendor{}+ v := &minuteVendor{clock: holdMinute(t)}_, gw := rpmGateway(t, v, 1)- if res, body, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {- t.Fatalf("first: %v %v %s", err, res, body)- }- began := time.Now()- res, body, err := chatOnce(context.Background(), gw)- if err != nil {- t.Fatal(err)+ clk := v.clock+ if r := clk.atOnce(t, "the first", askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 {+ t.Fatalf("first: %v %d %s", r.err, r.code, r.body)}- if res.StatusCode != 429 || res.Header.Get("Retry-After") != "2" || !strings.Contains(body, "1 requests a minute is its limit") {- t.Fatalf("second = %d Retry-After %q %s, want a 429 saying the limit, Retry-After 2", res.StatusCode, res.Header.Get("Retry-After"), body)+ // the clock held where the first went, the minute has room again in+ // the whole window, 1.5s: Retry-After 2+ r := clk.atOnce(t, "the second, to be turned away,", askRPM(context.Background(), gw, rlPath, rlBody))+ if r.err != nil {+ t.Fatal(r.err)}- if d := time.Since(began); d > window/2 {- t.Fatalf("turned away after %s, not at once", d)+ if r.code != 429 || r.header.Get("Retry-After") != "2" || !strings.Contains(r.body, "1 requests a minute is its limit") {+ t.Fatalf("second = %d Retry-After %q %s, want a 429 saying the limit, Retry-After 2", r.code, r.header.Get("Retry-After"), r.body)}if n := len(v.times()); n != 1 {t.Fatalf("the vendor saw %d, want 1", n)@@ -168,9 +366,9 @@ func TestMaxRPMTooLongIs429WithRetryAfter(t *testing.T) {if r := restingNow(t); len(r) != 0 {t.Fatalf("%v rest for a limit of the user's", r)}- time.Sleep(window)- if res, body, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {- t.Fatalf("after the minute: %v %v %s", err, res, body)+ clk.advance(window)+ if r := clk.atOnce(t, "the next, after the minute,", askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 {+ t.Fatalf("after the minute: %v %d %s", r.err, r.code, r.body)}}@@ -178,25 +376,29 @@ func TestMaxRPMTooLongIs429WithRetryAfter(t *testing.T) {// nothing is sent, and its place in the minute is given back.func TestMaxRPMAgentGoneWhileWaiting(t *testing.T) {shortMinute(t, 3*time.Second, time.Minute)- v := &minuteVendor{}+ v := &minuteVendor{clock: holdMinute(t)}s, gw := rpmGateway(t, v, 1)- if res, _, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {- t.Fatalf("first: %v %v", err, res)+ clk := v.clock+ if r := clk.atOnce(t, "the first", askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 {+ t.Fatalf("first: %v %d", r.err, r.code)}who := meterWho(provider.Provider{ID: "rl", Key: "k"})ctx, cancel := context.WithCancel(context.Background())- done := make(chan error, 1)- go func() {- _, _, err := chatOnce(ctx, gw)- done <- err- }()- within(t, "the second waiting", func() bool { return s.rpms.used(who) == 2 })+ second := askRPM(ctx, gw, rlPath, rlBody)+ clk.waitsUntil(t, "the second")+ if n := s.rpms.used(who); n != 2 {+ t.Fatalf("%d counted with the second waiting, want 2", n)+ }cancel()- if err := <-done; err == nil {+ if r := rpmAnswered(t, "the canceled request", second); r.err == nil {t.Fatal("the canceled request answered")}- within(t, "its place given back", func() bool { return s.rpms.used(who) == 1 })- time.Sleep(100 * time.Millisecond)+ rpmEventually(t, "its place given back", func() bool { return s.rpms.used(who) == 1 })+ // its wait ended with its place given back, so nothing is left to send+ // it once the minute has room+ if w := clk.waiting(); len(w) != 0 {+ t.Fatalf("%d still waiting for room after the agent went", len(w))+ }if n := len(v.times()); n != 1 {t.Fatalf("the vendor saw %d, want 1", n)}@@ -209,47 +411,41 @@ func TestMaxRPMAgentGoneWhileWaiting(t *testing.T) {func TestMaxRPMCountsRetriesAndCounts(t *testing.T) {window := 1500 * time.MillisecondslackMinute(t, window, time.Minute)- v := &minuteVendor{fail: 1, late: 150 * time.Millisecond}+ v := &minuteVendor{fail: 1, late: 150 * time.Millisecond, clock: holdMinute(t)}_, gw := rpmGateway(t, v, 3)- if res, body, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {- t.Fatalf("first: %v %v %s", err, res, body)+ clk := v.clock+ t0 := clk.Now()+ if r := clk.atOnce(t, "the first", askRPM(context.Background(), gw, rlPath, rlBody)); r.err != nil || r.code != 200 {+ t.Fatalf("first: %v %d %s", r.err, r.code, r.body)}if n := len(v.times()); n != 2 {t.Fatalf("the vendor saw %d, want the 503 and its retry", n)}- req, _ := http.NewRequest("POST", gw+"/v1/messages/count_tokens", strings.NewReader(`{"model":"rl/m","messages":[{"role":"user","content":"hi"}]}`))- req.Header.Set("Content-Type", "application/json")- res, err := http.DefaultClient.Do(req)- if err != nil {- t.Fatal(err)- }+ r := clk.atOnce(t, "the count, with room in the minute,", askRPM(context.Background(), gw, "/v1/messages/count_tokens", rlBody))var count struct {InputTokens int `json:"input_tokens"`}- json.NewDecoder(res.Body).Decode(&count)- res.Body.Close()- if count.InputTokens != 7 {- t.Fatalf("count = %d, want the vendor's 7", count.InputTokens)+ json.Unmarshal([]byte(r.body), &count)+ if r.err != nil || count.InputTokens != 7 {+ t.Fatalf("count = %d (%v %s), want the vendor's 7", count.InputTokens, r.err, r.body)}// with no room in the minute, a count doesn't wait: the estimate// answers, and the vendor isn't asked- req, _ = http.NewRequest("POST", gw+"/v1/messages/count_tokens", strings.NewReader(`{"model":"rl/m","messages":[{"role":"user","content":"hi"}]}`))- req.Header.Set("Content-Type", "application/json")- began := time.Now()- res, err = http.DefaultClient.Do(req)- if err != nil {- t.Fatal(err)- }- io.Copy(io.Discard, res.Body)- res.Body.Close()- if res.StatusCode != 200 || time.Since(began) > window/2 {- t.Fatalf("a count with no room: %d after %s", res.StatusCode, time.Since(began))+ r = clk.atOnce(t, "a count with no room", askRPM(context.Background(), gw, "/v1/messages/count_tokens", rlBody))+ if r.err != nil || r.code != 200 {+ t.Fatalf("a count with no room: %v %d %s", r.err, r.code, r.body)}if n := len(v.times()); n != 3 {t.Fatalf("the vendor saw %d, want the count estimated", n)}- if res, body, err := chatOnce(context.Background(), gw); err != nil || res.StatusCode != 200 {- t.Fatalf("last: %v %v %s", err, res, body)+ last := askRPM(context.Background(), gw, rlPath, rlBody)+ until := clk.waitsUntil(t, "the last")+ if want := t0.Add(window + rpmSlack); !until.Equal(want) {+ t.Fatalf("the last waits to go %s after the first, want the window and slack, %s: the retry or the count wasn't counted", until.Sub(t0), want.Sub(t0))+ }+ clk.advance(until.Sub(clk.Now()))+ if r := rpmAnswered(t, "the last", last); r.err != nil || r.code != 200 {+ t.Fatalf("last: %v %d %s", r.err, r.code, r.body)}at := v.times()if len(at) != 4 {