Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: large request bodies wait their turn for a share of memory (#4004)

yetone0c17aac2d701
diff --git a/internal/gateway/body_room.go b/internal/gateway/body_room.go
new file mode 100644
index 000000000..897dcb725
--- /dev/null
+++ b/internal/gateway/body_room.go
@@ -0,0 +1,180 @@
+package gateway
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "runtime"
+ "slices"
+ "sync"
+ "time"
+
+ "github.com/yetone/magpie/internal/proc"
+)
+
+// bodyRoom bounds the decoded request bytes the gateway serves at once.
+// One request's limit (defaultBodyLimit) bounds no sum: each body is held
+// while its request is served, with what is made from it, the model
+// rewritten, a translation to Chat, a plugin's base64 line, several times
+// its size. On a 2 GiB server Codex turns of ~45 MiB, one after another
+// and side by side, held the Go heap at ~500 MiB and the plugin host at
+// ~500 MiB, into swap (jihuayu, #4004).
+//
+// A body of bodyRoomFree or more takes room for its size until its
+// request ends. With too little left it waits its turn, first come first
+// served, for requests to end, bodyRoomWait at most, and is answered 503
+// with the provider not asked. A body bigger than the whole room runs
+// alone. Small bodies, an agent's ordinary turns, pass without counting.
+type bodyRoomT struct {
+ mu sync.Mutex
+ // room is the most held at once; 0 no bound. sized says room was set.
+ room int64
+ why string
+ sized bool
+ held int64
+ // by is what each request being served holds, by its context, so a
+ // body read twice in one request takes its room once
+ by map[context.Context]int64
+ queue []*bodyWaiter
+}
+
+type bodyWaiter struct {
+ ctx context.Context
+ n int64
+ in chan struct{} // closed when let in
+}
+
+var bodyRoom = &bodyRoomT{}
+
+// bodyRoomFree is the size a body takes room from; smaller ones pass.
+const bodyRoomFree = 1 << 20
+
+// bodyRoomWait is how long a body waits for room before its request is
+// answered 503: within unroutedMost, which the wait counts toward
+// (noteWait), so the answer says why.
+var bodyRoomWait = 100 * time.Second
+
+// bodyRoomFloor is the least room, however small the machine.
+const bodyRoomFloor = 64 << 20
+
+// bodyRoomSize is a thirty-second of what magpie's processes may use
+// (memLimit on Linux, the computer's memory elsewhere), bodyRoomFloor at
+// least: a 44 MiB Responses turn sent on as Chat peaked at ~700 MiB of Go
+// heap on its way, garbage not yet collected included, and a plugin's host
+// holds more. Where neither is known there is no bound: a limit not known
+// is not a small one.
+func bodyRoomSize() (int64, string) {
+ limit, why := memLimit(memGOOS, memRoot)
+ if limit <= 0 && memGOOS == runtime.GOOS {
+ limit, why = proc.MemTotal(), "the computer's memory"
+ }
+ if limit <= 0 {
+ return 0, ""
+ }
+ return max(limit/32, bodyRoomFloor), fmt.Sprintf("a thirty-second of %d MiB (%s)", limit>>20, why)
+}
+
+// bodyFull is the error of a body that found no room in bodyRoomWait.
+type bodyFull struct {
+ n, held, room int64
+ why string
+ waited time.Duration
+}
+
+func (e *bodyFull) Error() string {
+ return fmt.Sprintf("magpie is serving %d MiB of request bodies at once, as much as it holds (%d MiB, %s), "+
+ "and none of them ended within %s to make room for this %d MiB one; the provider was not asked",
+ e.held>>20, e.room>>20, e.why, e.waited, e.n>>20)
+}
+
+// take holds n bytes of room for the request ctx belongs to until ctx is
+// done, waiting its turn for it. A request takes it on its Content-Length
+// before its body is read, so the ones waiting don't hold theirs, and
+// again on the body read, which only adds what that is over the first. A
+// ctx that is never done (a caller's own) takes nothing, since nothing
+// would give it back.
+func (b *bodyRoomT) take(ctx context.Context, n int64) error {
+ if n < bodyRoomFree || ctx.Done() == nil {
+ return nil
+ }
+ b.mu.Lock()
+ if !b.sized {
+ b.sized = true
+ b.room, b.why = bodyRoomSize()
+ }
+ if h, ok := b.by[ctx]; ok || b.room <= 0 {
+ if ok && n > h {
+ // the body read is bigger than its request said, a compressed
+ // one: it holds what it is, let in already
+ b.held += n - h
+ b.by[ctx] = n
+ }
+ b.mu.Unlock()
+ return nil
+ }
+ w := &bodyWaiter{ctx: ctx, n: n, in: make(chan struct{})}
+ b.queue = append(b.queue, w)
+ b.letIn()
+ held, room, why := b.held, b.room, b.why
+ b.mu.Unlock()
+ select {
+ case <-w.in:
+ return nil
+ default:
+ }
+ start := time.Now()
+ log.Printf("gateway: a %d MiB request body waits for room: %d MiB of %d MiB held (%s)", n>>20, held>>20, room>>20, why)
+ defer noteWait(ctx)()
+ timer := time.NewTimer(bodyRoomWait)
+ defer timer.Stop()
+ var err error
+ select {
+ case <-w.in:
+ return nil
+ case <-ctx.Done():
+ err = ctx.Err()
+ case <-timer.C:
+ }
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ select {
+ case <-w.in:
+ return nil // let in as it gave up: it holds the room, and ctx gives it back
+ default:
+ }
+ b.queue = slices.DeleteFunc(b.queue, func(q *bodyWaiter) bool { return q == w })
+ b.letIn() // one ahead gone may let the next in
+ if err != nil {
+ return err
+ }
+ return &bodyFull{n: n, held: b.held, room: b.room, why: b.why, waited: time.Since(start).Round(time.Second)}
+}
+
+// letIn lets in, in order, each waiter there is room for: one bigger than
+// the room when nothing is held. b.mu is held.
+func (b *bodyRoomT) letIn() {
+ for len(b.queue) > 0 {
+ w := b.queue[0]
+ if b.held > 0 && b.held+w.n > b.room {
+ return
+ }
+ b.queue = b.queue[1:]
+ if b.by == nil {
+ b.by = map[context.Context]int64{}
+ }
+ b.held += w.n
+ b.by[w.ctx] = w.n
+ close(w.in)
+ ctx := w.ctx
+ context.AfterFunc(ctx, func() { b.give(ctx) })
+ }
+}
+
+// give returns what ctx's request held.
+func (b *bodyRoomT) give(ctx context.Context) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ b.held -= b.by[ctx]
+ delete(b.by, ctx)
+ b.letIn()
+}
diff --git a/internal/gateway/body_room_test.go b/internal/gateway/body_room_test.go
new file mode 100644
index 000000000..3bc556dfe
--- /dev/null
+++ b/internal/gateway/body_room_test.go
@@ -0,0 +1,266 @@
+package gateway
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/yetone/magpie/internal/provider"
+)
+
+func roomOf(room int64) *bodyRoomT { return &bodyRoomT{room: room, why: "test", sized: true} }
+
+// taking runs b.take in the background; its result comes on the channel.
+func taking(b *bodyRoomT, ctx context.Context, n int64) chan error {
+ out := make(chan error, 1)
+ go func() { out <- b.take(ctx, n) }()
+ return out
+}
+
+func waitQueued(t *testing.T, b *bodyRoomT, n int) {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ b.mu.Lock()
+ q := len(b.queue)
+ b.mu.Unlock()
+ if q == n {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("queue = %d, want %d", q, n)
+ }
+ time.Sleep(time.Millisecond)
+ }
+}
+
+func roomHeld(b *bodyRoomT) int64 {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ return b.held
+}
+
+// Bodies that fit are let in at once; the next waits until one of the
+// requests holding room ends, then holds its own, and every byte is given
+// back when the requests end (#4004).
+func TestBodyRoomWaitsForARequestToEnd(t *testing.T) {
+ b := roomOf(100 << 20)
+ ctx1, end1 := context.WithCancel(context.Background())
+ ctx2, end2 := context.WithCancel(context.Background())
+ ctx3, end3 := context.WithCancel(context.Background())
+ defer end2()
+ defer end3()
+ if err := b.take(ctx1, 45<<20); err != nil {
+ t.Fatal(err)
+ }
+ if err := b.take(ctx2, 45<<20); err != nil {
+ t.Fatal(err)
+ }
+ third := taking(b, ctx3, 45<<20)
+ waitQueued(t, b, 1)
+ select {
+ case err := <-third:
+ t.Fatalf("a third 45 MiB body was let into 100 MiB beside two: %v", err)
+ default:
+ }
+ end1()
+ if err := <-third; err != nil {
+ t.Fatal(err)
+ }
+ if h := roomHeld(b); h != 90<<20 {
+ t.Fatalf("held = %d MiB, want 90", h>>20)
+ }
+ end2()
+ end3()
+ deadline := time.Now().Add(10 * time.Second)
+ for roomHeld(b) != 0 {
+ if time.Now().After(deadline) {
+ t.Fatalf("held = %d after every request ended", roomHeld(b))
+ }
+ time.Sleep(time.Millisecond)
+ }
+}
+
+// A body bigger than the room runs alone: let in when nothing is held, and
+// it waits, rather than being refused, while something is.
+func TestBodyRoomBiggerThanTheRoomRunsAlone(t *testing.T) {
+ b := roomOf(64 << 20)
+ small, endSmall := context.WithCancel(context.Background())
+ big, endBig := context.WithCancel(context.Background())
+ defer endBig()
+ if err := b.take(small, 2<<20); err != nil {
+ t.Fatal(err)
+ }
+ got := taking(b, big, 200<<20)
+ waitQueued(t, b, 1)
+ endSmall()
+ if err := <-got; err != nil {
+ t.Fatal(err)
+ }
+ if h := roomHeld(b); h != 200<<20 {
+ t.Fatalf("held = %d MiB, want 200", h>>20)
+ }
+}
+
+// First come, first served: a small body that came after a waiting big one
+// waits behind it, so a big body isn't passed by a stream of small ones.
+func TestBodyRoomKeepsOrder(t *testing.T) {
+ b := roomOf(64 << 20)
+ first, endFirst := context.WithCancel(context.Background())
+ big, endBig := context.WithCancel(context.Background())
+ later, endLater := context.WithCancel(context.Background())
+ defer endBig()
+ defer endLater()
+ if err := b.take(first, 40<<20); err != nil {
+ t.Fatal(err)
+ }
+ bigGot := taking(b, big, 40<<20)
+ waitQueued(t, b, 1)
+ laterGot := taking(b, later, 2<<20)
+ waitQueued(t, b, 2)
+ select {
+ case <-laterGot:
+ t.Fatal("a 2 MiB body went ahead of the 40 MiB one waiting before it")
+ default:
+ }
+ endFirst()
+ if err := <-bigGot; err != nil {
+ t.Fatal(err)
+ }
+ if err := <-laterGot; err != nil {
+ t.Fatal(err)
+ }
+}
+
+// A body that waits bodyRoomWait is refused with what is held and why, and
+// leaves the queue, so the one behind it is let in when room comes.
+func TestBodyRoomRefusesAfterTheWait(t *testing.T) {
+ was := bodyRoomWait
+ bodyRoomWait = 50 * time.Millisecond
+ defer func() { bodyRoomWait = was }()
+ b := roomOf(64 << 20)
+ ctx1, end1 := context.WithCancel(context.Background())
+ ctx2, end2 := context.WithCancel(context.Background())
+ defer end1()
+ defer end2()
+ if err := b.take(ctx1, 60<<20); err != nil {
+ t.Fatal(err)
+ }
+ err := b.take(ctx2, 30<<20)
+ var full *bodyFull
+ if !errors.As(err, &full) || !strings.Contains(err.Error(), "60 MiB") || !strings.Contains(err.Error(), "the provider was not asked") {
+ t.Fatalf("err = %v", err)
+ }
+ b.mu.Lock()
+ q := len(b.queue)
+ b.mu.Unlock()
+ if q != 0 || roomHeld(b) != 60<<20 {
+ t.Fatalf("queue = %d, held = %d MiB after a refusal", q, roomHeld(b)>>20)
+ }
+}
+
+// A waiter whose client went away leaves the queue and holds nothing.
+func TestBodyRoomGoneWaiterLeaves(t *testing.T) {
+ b := roomOf(64 << 20)
+ ctx1, end1 := context.WithCancel(context.Background())
+ ctx2, end2 := context.WithCancel(context.Background())
+ defer end1()
+ if err := b.take(ctx1, 60<<20); err != nil {
+ t.Fatal(err)
+ }
+ got := taking(b, ctx2, 30<<20)
+ waitQueued(t, b, 1)
+ end2()
+ if err := <-got; !errors.Is(err, context.Canceled) {
+ t.Fatalf("err = %v", err)
+ }
+ if h := roomHeld(b); h != 60<<20 {
+ t.Fatalf("held = %d MiB", h>>20)
+ }
+}
+
+// Small bodies, a context never done, a request reading its body twice
+// and a machine whose memory isn't known take no room.
+func TestBodyRoomTakesNothingItCantGiveBack(t *testing.T) {
+ b := roomOf(64 << 20)
+ ctx, end := context.WithCancel(context.Background())
+ defer end()
+ if err := b.take(ctx, bodyRoomFree-1); err != nil || roomHeld(b) != 0 {
+ t.Fatalf("a small body: %v, held %d", err, roomHeld(b))
+ }
+ if err := b.take(context.Background(), 100<<20); err != nil || roomHeld(b) != 0 {
+ t.Fatalf("a context never done: %v, held %d", err, roomHeld(b))
+ }
+ if err := b.take(ctx, 40<<20); err != nil {
+ t.Fatal(err)
+ }
+ if err := b.take(ctx, 40<<20); err != nil || roomHeld(b) != 40<<20 {
+ t.Fatalf("one request's second read: %v, held %d MiB", err, roomHeld(b)>>20)
+ }
+ if err := b.take(ctx, 50<<20); err != nil || roomHeld(b) != 50<<20 {
+ t.Fatalf("a body read bigger than its request said: %v, held %d MiB", err, roomHeld(b)>>20)
+ }
+ unknown := roomOf(0)
+ if err := unknown.take(ctx, 400<<20); err != nil || roomHeld(unknown) != 0 {
+ t.Fatalf("memory not known: %v, held %d", err, roomHeld(unknown))
+ }
+}
+
+// The room is a thirty-second of the memory magpie may use, 64 MiB at
+// least: jihuayu's 1,670 MiB server serves one ~45 MiB body at a time.
+func TestBodyRoomSize(t *testing.T) {
+ root := t.TempDir()
+ writeMeminfo := func(kb string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Join(root, "proc"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(root, "proc/meminfo"), []byte("MemTotal: "+kb+" kB\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ wasGOOS, wasRoot := memGOOS, memRoot
+ memGOOS, memRoot = "linux", root
+ defer func() { memGOOS, memRoot = wasGOOS, wasRoot }()
+ writeMeminfo("1710080") // 1,670 MiB
+ if n, why := bodyRoomSize(); n != bodyRoomFloor || !strings.Contains(why, "1670 MiB") {
+ t.Fatalf("1,670 MiB: %d MiB (%s), want the floor", n>>20, why)
+ }
+ writeMeminfo("8388608") // 8 GiB
+ if n, _ := bodyRoomSize(); n != 256<<20 {
+ t.Fatalf("8 GiB: %d MiB, want 256", n>>20)
+ }
+}
+
+// Through the gateway: a request whose body finds no room is answered 503
+// in its protocol's envelope, and the provider is not asked.
+func TestBodyRoomFullAnswers503(t *testing.T) {
+ was, wasWait := bodyRoom, bodyRoomWait
+ bodyRoom, bodyRoomWait = roomOf(64<<20), 50*time.Millisecond
+ defer func() { bodyRoom, bodyRoomWait = was, wasWait }()
+ hold, end := context.WithCancel(context.Background())
+ defer end()
+ if err := bodyRoom.take(hold, 60<<20); err != nil {
+ t.Fatal(err)
+ }
+ s := &Server{}
+ body := `{"model":"m","input":"` + strings.Repeat("x", 30<<20) + `"}`
+ r := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader([]byte(body)))
+ ctx, cancel := context.WithCancel(r.Context())
+ defer cancel()
+ r = r.WithContext(ctx)
+ w := httptest.NewRecorder()
+ if got, ok := s.requestBody(w, r, provider.Responses); ok || got != nil {
+ t.Fatal("a body with no room was read through")
+ }
+ if w.Code != http.StatusServiceUnavailable || !strings.Contains(w.Body.String(), "the provider was not asked") {
+ t.Fatalf("%d %s", w.Code, w.Body.String())
+ }
+}
diff --git a/internal/gateway/request_bounds.go b/internal/gateway/request_bounds.go
index 16d836f73..d0db1bf13 100644
--- a/internal/gateway/request_bounds.go
+++ b/internal/gateway/request_bounds.go
@@ -44,11 +44,27 @@ func (s *Server) readRequestBody(w http.ResponseWriter, r *http.Request, from pr
if limit > 0 {
limits.body = limit
}
+ // wait for room before reading: a body waiting is not held (bodyRoom)
+ most := limits.body
+ if most <= 0 {
+ most = defaultBodyLimit
+ }
+ if n := r.ContentLength; n > 0 && n <= most {
+ if err := bodyRoom.take(r.Context(), n); err != nil {
+ w.Header().Set("Connection", "close") // its body is left unread
+ writeError(w, from, http.StatusServiceUnavailable, err.Error())
+ return nil, false
+ }
+ }
body, status, err := readBoundedRequestBody(w, r, limits)
if err != nil {
writeError(w, from, status, err.Error())
return nil, false
}
+ if err := bodyRoom.take(r.Context(), int64(len(body))); err != nil {
+ writeError(w, from, http.StatusServiceUnavailable, err.Error())
+ return nil, false
+ }
holdBody(r.Context(), body)
return body, true
}