Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: once it rests 30s, what its requests left goes back to the system (#3472)
yetone20d63621d12c
diff --git a/internal/gateway/busy.go b/internal/gateway/busy.goindex 4e7946e4a..b2edb7804 100644--- a/internal/gateway/busy.go+++ b/internal/gateway/busy.go@@ -74,7 +74,9 @@ func (s *Server) counted(h http.Handler) http.Handler {}done()s.lastDone.Store(time.Now().UnixNano())- s.inFlight.Add(-1)+ if s.inFlight.Add(-1) == 0 {+ s.armRelease(s.idle.restFor())+ }}()h.ServeHTTP(w, r)})@@ -88,6 +90,8 @@ type busyCounters struct {// the subscription bridge's turns among them, and when the last endedbridgeInFlight atomic.Int64bridgeDone atomic.Int64 // unix ns+ // handing back what requests left once it rests (idle_memory.go)+ idle idleMemory}// parked is how many runs wait on their caller's tool results.diff --git a/internal/gateway/idle_memory.go b/internal/gateway/idle_memory.gonew file mode 100644index 000000000..136472d1c--- /dev/null+++ b/internal/gateway/idle_memory.go@@ -0,0 +1,75 @@+package gateway++import (+ "runtime/debug"+ "sync"+ "time"+)++// A request's body is held several times over while it is served (as+// read, masked, captured for the log), and Go keeps the pages it freed+// after: a tray magpie with ~60 requests an hour sat at 725 MB of private+// working set after a collection, its heap holding 1088 MB of which 315 MB+// was live (#3472). Go's scavenger hands such pages back only slowly, and+// not while the next collection's goal covers them. So once the gateway+// has had nothing in flight for idleRelease, what its requests left is+// handed back at once, as the warm-up does with what its reads left.++// idleRelease is how long the gateway rests before that.+const idleRelease = 30 * time.Second++// idleMemory is a Server's release of idle memory: a test sets its rest+// and counts its releases.+type idleMemory struct {+ mu sync.Mutex+ due *time.Timer // the release to come, or nil+ rest time.Duration // idleRelease when zero+ release func() // debug.FreeOSMemory when nil+}++// stop calls off the release to come.+func (m *idleMemory) stop() {+ m.mu.Lock()+ defer m.mu.Unlock()+ if m.due != nil {+ m.due.Stop()+ m.due = nil+ }+}++func (m *idleMemory) restFor() time.Duration {+ if m.rest > 0 {+ return m.rest+ }+ return idleRelease+}++// armRelease has releaseIfResting run after d, unless it is due already.+func (s *Server) armRelease(d time.Duration) {+ s.idle.mu.Lock()+ defer s.idle.mu.Unlock()+ if s.idle.due == nil {+ s.idle.due = time.AfterFunc(d, s.releaseIfResting)+ }+}++// releaseIfResting hands memory back when nothing has been in flight for+// idleRelease. A request in flight now arms it again when it ends, and a+// rest not long enough yet waits out the rest of it.+func (s *Server) releaseIfResting() {+ s.idle.mu.Lock()+ s.idle.due = nil+ s.idle.mu.Unlock()+ if s.inFlight.Load() != 0 {+ return+ }+ if wait := s.idle.restFor() - time.Since(time.Unix(0, s.lastDone.Load())); wait > 0 {+ s.armRelease(wait)+ return+ }+ if s.idle.release != nil {+ s.idle.release()+ return+ }+ debug.FreeOSMemory()+}diff --git a/internal/gateway/idle_memory_test.go b/internal/gateway/idle_memory_test.gonew file mode 100644index 000000000..b1d5065e0--- /dev/null+++ b/internal/gateway/idle_memory_test.go@@ -0,0 +1,75 @@+package gateway++import (+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++// Once the gateway rests, what its requests left in the heap goes back to+// the system, once per rest, and never while a request is in flight+// (#3472).+func TestRestingGatewayHandsMemoryBack(t *testing.T) {+ fresh(t)+ s := New()+ var released atomic.Int32+ s.idle.rest = 20 * time.Millisecond+ s.idle.release = func() { released.Add(1) }+ t.Cleanup(s.idle.stop)+ hold, held := make(chan struct{}), make(chan struct{})+ h := s.counted(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/hold" {+ close(held)+ <-hold+ }+ }))+ post := func(path string) {+ h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", path, strings.NewReader(`{}`)))+ }+ waitReleased := func(n int32) {+ t.Helper()+ for deadline := time.Now().Add(10 * time.Second); released.Load() < n; {+ if time.Now().After(deadline) {+ t.Fatalf("released %d times, want %d", released.Load(), n)+ }+ time.Sleep(5 * time.Millisecond)+ }+ }++ post("/v1/messages")+ waitReleased(1)++ // a request in flight: the rest it would end is not one+ go post("/hold")+ <-held+ s.releaseIfResting()+ if n := released.Load(); n != 1 {+ t.Fatalf("released %d times with a request in flight, want 1", n)+ }+ // a request just ended: the rest is not long enough yet+ s.idle.rest = time.Hour+ post("/v1/messages")+ s.idle.stop()+ s.releaseIfResting()+ if n := released.Load(); n != 1 {+ t.Fatalf("released %d times right after a request, want 1", n)+ }+ s.idle.stop()+ s.idle.rest = 20 * time.Millisecond+ close(hold)+ for s.Busy().Requests != 0 {+ time.Sleep(time.Millisecond)+ }+ waitReleased(2)+ // a GET (the model list another magpie asks for) is not work to rest from+ h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/v1/models", nil))+ s.idle.mu.Lock()+ armed := s.idle.due != nil+ s.idle.mu.Unlock()+ if armed {+ t.Fatal("a GET armed a release")+ }+}