Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: once it rests 30s, what its requests left goes back to the system (#3472)

yetone20d63621d12c
diff --git a/internal/gateway/busy.go b/internal/gateway/busy.go
index 4e7946e4a..b2edb7804 100644
--- a/internal/gateway/busy.go
+++ b/internal/gateway/busy.go
@@ -74,7 +74,9 @@ func (s *Server) counted(h http.Handler) http.Handler {
}
done()
s.lastDone.Store(time.Now().UnixNano())
- s.inFlight.Add(-1)
+ if s.inFlight.Add(-1) == 0 {
+ s.armRelease(s.idle.restFor())
+ }
}()
h.ServeHTTP(w, r)
})
@@ -88,6 +90,8 @@ type busyCounters struct {
// the subscription bridge's turns among them, and when the last ended
bridgeInFlight atomic.Int64
bridgeDone atomic.Int64 // unix ns
+ // handing back what requests left once it rests (idle_memory.go)
+ idle idleMemory
}
// parked is how many runs wait on their caller's tool results.
diff --git a/internal/gateway/idle_memory.go b/internal/gateway/idle_memory.go
new file mode 100644
index 000000000..136472d1c
--- /dev/null
+++ b/internal/gateway/idle_memory.go
@@ -0,0 +1,75 @@
+package gateway
+
+import (
+ "runtime/debug"
+ "sync"
+ "time"
+)
+
+// A request's body is held several times over while it is served (as
+// read, masked, captured for the log), and Go keeps the pages it freed
+// after: a tray magpie with ~60 requests an hour sat at 725 MB of private
+// working set after a collection, its heap holding 1088 MB of which 315 MB
+// was live (#3472). Go's scavenger hands such pages back only slowly, and
+// not while the next collection's goal covers them. So once the gateway
+// has had nothing in flight for idleRelease, what its requests left is
+// handed back at once, as the warm-up does with what its reads left.
+
+// idleRelease is how long the gateway rests before that.
+const idleRelease = 30 * time.Second
+
+// idleMemory is a Server's release of idle memory: a test sets its rest
+// and counts its releases.
+type idleMemory struct {
+ mu sync.Mutex
+ due *time.Timer // the release to come, or nil
+ rest time.Duration // idleRelease when zero
+ release func() // debug.FreeOSMemory when nil
+}
+
+// stop calls off the release to come.
+func (m *idleMemory) stop() {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.due != nil {
+ m.due.Stop()
+ m.due = nil
+ }
+}
+
+func (m *idleMemory) restFor() time.Duration {
+ if m.rest > 0 {
+ return m.rest
+ }
+ return idleRelease
+}
+
+// armRelease has releaseIfResting run after d, unless it is due already.
+func (s *Server) armRelease(d time.Duration) {
+ s.idle.mu.Lock()
+ defer s.idle.mu.Unlock()
+ if s.idle.due == nil {
+ s.idle.due = time.AfterFunc(d, s.releaseIfResting)
+ }
+}
+
+// releaseIfResting hands memory back when nothing has been in flight for
+// idleRelease. A request in flight now arms it again when it ends, and a
+// rest not long enough yet waits out the rest of it.
+func (s *Server) releaseIfResting() {
+ s.idle.mu.Lock()
+ s.idle.due = nil
+ s.idle.mu.Unlock()
+ if s.inFlight.Load() != 0 {
+ return
+ }
+ if wait := s.idle.restFor() - time.Since(time.Unix(0, s.lastDone.Load())); wait > 0 {
+ s.armRelease(wait)
+ return
+ }
+ if s.idle.release != nil {
+ s.idle.release()
+ return
+ }
+ debug.FreeOSMemory()
+}
diff --git a/internal/gateway/idle_memory_test.go b/internal/gateway/idle_memory_test.go
new file mode 100644
index 000000000..b1d5065e0
--- /dev/null
+++ b/internal/gateway/idle_memory_test.go
@@ -0,0 +1,75 @@
+package gateway
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// Once the gateway rests, what its requests left in the heap goes back to
+// the system, once per rest, and never while a request is in flight
+// (#3472).
+func TestRestingGatewayHandsMemoryBack(t *testing.T) {
+ fresh(t)
+ s := New()
+ var released atomic.Int32
+ s.idle.rest = 20 * time.Millisecond
+ s.idle.release = func() { released.Add(1) }
+ t.Cleanup(s.idle.stop)
+ hold, held := make(chan struct{}), make(chan struct{})
+ h := s.counted(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/hold" {
+ close(held)
+ <-hold
+ }
+ }))
+ post := func(path string) {
+ h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", path, strings.NewReader(`{}`)))
+ }
+ waitReleased := func(n int32) {
+ t.Helper()
+ for deadline := time.Now().Add(10 * time.Second); released.Load() < n; {
+ if time.Now().After(deadline) {
+ t.Fatalf("released %d times, want %d", released.Load(), n)
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ }
+
+ post("/v1/messages")
+ waitReleased(1)
+
+ // a request in flight: the rest it would end is not one
+ go post("/hold")
+ <-held
+ s.releaseIfResting()
+ if n := released.Load(); n != 1 {
+ t.Fatalf("released %d times with a request in flight, want 1", n)
+ }
+ // a request just ended: the rest is not long enough yet
+ s.idle.rest = time.Hour
+ post("/v1/messages")
+ s.idle.stop()
+ s.releaseIfResting()
+ if n := released.Load(); n != 1 {
+ t.Fatalf("released %d times right after a request, want 1", n)
+ }
+ s.idle.stop()
+ s.idle.rest = 20 * time.Millisecond
+ close(hold)
+ for s.Busy().Requests != 0 {
+ time.Sleep(time.Millisecond)
+ }
+ waitReleased(2)
+ // a GET (the model list another magpie asks for) is not work to rest from
+ h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/v1/models", nil))
+ s.idle.mu.Lock()
+ armed := s.idle.due != nil
+ s.idle.mu.Unlock()
+ if armed {
+ t.Fatal("a GET armed a release")
+ }
+}