Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: large request bodies wait their turn for a share of memory (#4004)
yetone0c17aac2d701
diff --git a/internal/gateway/body_room.go b/internal/gateway/body_room.gonew file mode 100644index 000000000..897dcb725--- /dev/null+++ b/internal/gateway/body_room.go@@ -0,0 +1,180 @@+package gateway++import (+ "context"+ "fmt"+ "log"+ "runtime"+ "slices"+ "sync"+ "time"++ "github.com/yetone/magpie/internal/proc"+)++// bodyRoom bounds the decoded request bytes the gateway serves at once.+// One request's limit (defaultBodyLimit) bounds no sum: each body is held+// while its request is served, with what is made from it, the model+// rewritten, a translation to Chat, a plugin's base64 line, several times+// its size. On a 2 GiB server Codex turns of ~45 MiB, one after another+// and side by side, held the Go heap at ~500 MiB and the plugin host at+// ~500 MiB, into swap (jihuayu, #4004).+//+// A body of bodyRoomFree or more takes room for its size until its+// request ends. With too little left it waits its turn, first come first+// served, for requests to end, bodyRoomWait at most, and is answered 503+// with the provider not asked. A body bigger than the whole room runs+// alone. Small bodies, an agent's ordinary turns, pass without counting.+type bodyRoomT struct {+ mu sync.Mutex+ // room is the most held at once; 0 no bound. sized says room was set.+ room int64+ why string+ sized bool+ held int64+ // by is what each request being served holds, by its context, so a+ // body read twice in one request takes its room once+ by map[context.Context]int64+ queue []*bodyWaiter+}++type bodyWaiter struct {+ ctx context.Context+ n int64+ in chan struct{} // closed when let in+}++var bodyRoom = &bodyRoomT{}++// bodyRoomFree is the size a body takes room from; smaller ones pass.+const bodyRoomFree = 1 << 20++// bodyRoomWait is how long a body waits for room before its request is+// answered 503: within unroutedMost, which the wait counts toward+// (noteWait), so the answer says why.+var bodyRoomWait = 100 * time.Second++// bodyRoomFloor is the least room, however small the machine.+const bodyRoomFloor = 64 << 20++// bodyRoomSize is a thirty-second of what magpie's processes may use+// (memLimit on Linux, the computer's memory elsewhere), bodyRoomFloor at+// least: a 44 MiB Responses turn sent on as Chat peaked at ~700 MiB of Go+// heap on its way, garbage not yet collected included, and a plugin's host+// holds more. Where neither is known there is no bound: a limit not known+// is not a small one.+func bodyRoomSize() (int64, string) {+ limit, why := memLimit(memGOOS, memRoot)+ if limit <= 0 && memGOOS == runtime.GOOS {+ limit, why = proc.MemTotal(), "the computer's memory"+ }+ if limit <= 0 {+ return 0, ""+ }+ return max(limit/32, bodyRoomFloor), fmt.Sprintf("a thirty-second of %d MiB (%s)", limit>>20, why)+}++// bodyFull is the error of a body that found no room in bodyRoomWait.+type bodyFull struct {+ n, held, room int64+ why string+ waited time.Duration+}++func (e *bodyFull) Error() string {+ return fmt.Sprintf("magpie is serving %d MiB of request bodies at once, as much as it holds (%d MiB, %s), "++ "and none of them ended within %s to make room for this %d MiB one; the provider was not asked",+ e.held>>20, e.room>>20, e.why, e.waited, e.n>>20)+}++// take holds n bytes of room for the request ctx belongs to until ctx is+// done, waiting its turn for it. A request takes it on its Content-Length+// before its body is read, so the ones waiting don't hold theirs, and+// again on the body read, which only adds what that is over the first. A+// ctx that is never done (a caller's own) takes nothing, since nothing+// would give it back.+func (b *bodyRoomT) take(ctx context.Context, n int64) error {+ if n < bodyRoomFree || ctx.Done() == nil {+ return nil+ }+ b.mu.Lock()+ if !b.sized {+ b.sized = true+ b.room, b.why = bodyRoomSize()+ }+ if h, ok := b.by[ctx]; ok || b.room <= 0 {+ if ok && n > h {+ // the body read is bigger than its request said, a compressed+ // one: it holds what it is, let in already+ b.held += n - h+ b.by[ctx] = n+ }+ b.mu.Unlock()+ return nil+ }+ w := &bodyWaiter{ctx: ctx, n: n, in: make(chan struct{})}+ b.queue = append(b.queue, w)+ b.letIn()+ held, room, why := b.held, b.room, b.why+ b.mu.Unlock()+ select {+ case <-w.in:+ return nil+ default:+ }+ start := time.Now()+ log.Printf("gateway: a %d MiB request body waits for room: %d MiB of %d MiB held (%s)", n>>20, held>>20, room>>20, why)+ defer noteWait(ctx)()+ timer := time.NewTimer(bodyRoomWait)+ defer timer.Stop()+ var err error+ select {+ case <-w.in:+ return nil+ case <-ctx.Done():+ err = ctx.Err()+ case <-timer.C:+ }+ b.mu.Lock()+ defer b.mu.Unlock()+ select {+ case <-w.in:+ return nil // let in as it gave up: it holds the room, and ctx gives it back+ default:+ }+ b.queue = slices.DeleteFunc(b.queue, func(q *bodyWaiter) bool { return q == w })+ b.letIn() // one ahead gone may let the next in+ if err != nil {+ return err+ }+ return &bodyFull{n: n, held: b.held, room: b.room, why: b.why, waited: time.Since(start).Round(time.Second)}+}++// letIn lets in, in order, each waiter there is room for: one bigger than+// the room when nothing is held. b.mu is held.+func (b *bodyRoomT) letIn() {+ for len(b.queue) > 0 {+ w := b.queue[0]+ if b.held > 0 && b.held+w.n > b.room {+ return+ }+ b.queue = b.queue[1:]+ if b.by == nil {+ b.by = map[context.Context]int64{}+ }+ b.held += w.n+ b.by[w.ctx] = w.n+ close(w.in)+ ctx := w.ctx+ context.AfterFunc(ctx, func() { b.give(ctx) })+ }+}++// give returns what ctx's request held.+func (b *bodyRoomT) give(ctx context.Context) {+ b.mu.Lock()+ defer b.mu.Unlock()+ b.held -= b.by[ctx]+ delete(b.by, ctx)+ b.letIn()+}diff --git a/internal/gateway/body_room_test.go b/internal/gateway/body_room_test.gonew file mode 100644index 000000000..3bc556dfe--- /dev/null+++ b/internal/gateway/body_room_test.go@@ -0,0 +1,266 @@+package gateway++import (+ "bytes"+ "context"+ "errors"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "github.com/yetone/magpie/internal/provider"+)++func roomOf(room int64) *bodyRoomT { return &bodyRoomT{room: room, why: "test", sized: true} }++// taking runs b.take in the background; its result comes on the channel.+func taking(b *bodyRoomT, ctx context.Context, n int64) chan error {+ out := make(chan error, 1)+ go func() { out <- b.take(ctx, n) }()+ return out+}++func waitQueued(t *testing.T, b *bodyRoomT, n int) {+ t.Helper()+ deadline := time.Now().Add(10 * time.Second)+ for {+ b.mu.Lock()+ q := len(b.queue)+ b.mu.Unlock()+ if q == n {+ return+ }+ if time.Now().After(deadline) {+ t.Fatalf("queue = %d, want %d", q, n)+ }+ time.Sleep(time.Millisecond)+ }+}++func roomHeld(b *bodyRoomT) int64 {+ b.mu.Lock()+ defer b.mu.Unlock()+ return b.held+}++// Bodies that fit are let in at once; the next waits until one of the+// requests holding room ends, then holds its own, and every byte is given+// back when the requests end (#4004).+func TestBodyRoomWaitsForARequestToEnd(t *testing.T) {+ b := roomOf(100 << 20)+ ctx1, end1 := context.WithCancel(context.Background())+ ctx2, end2 := context.WithCancel(context.Background())+ ctx3, end3 := context.WithCancel(context.Background())+ defer end2()+ defer end3()+ if err := b.take(ctx1, 45<<20); err != nil {+ t.Fatal(err)+ }+ if err := b.take(ctx2, 45<<20); err != nil {+ t.Fatal(err)+ }+ third := taking(b, ctx3, 45<<20)+ waitQueued(t, b, 1)+ select {+ case err := <-third:+ t.Fatalf("a third 45 MiB body was let into 100 MiB beside two: %v", err)+ default:+ }+ end1()+ if err := <-third; err != nil {+ t.Fatal(err)+ }+ if h := roomHeld(b); h != 90<<20 {+ t.Fatalf("held = %d MiB, want 90", h>>20)+ }+ end2()+ end3()+ deadline := time.Now().Add(10 * time.Second)+ for roomHeld(b) != 0 {+ if time.Now().After(deadline) {+ t.Fatalf("held = %d after every request ended", roomHeld(b))+ }+ time.Sleep(time.Millisecond)+ }+}++// A body bigger than the room runs alone: let in when nothing is held, and+// it waits, rather than being refused, while something is.+func TestBodyRoomBiggerThanTheRoomRunsAlone(t *testing.T) {+ b := roomOf(64 << 20)+ small, endSmall := context.WithCancel(context.Background())+ big, endBig := context.WithCancel(context.Background())+ defer endBig()+ if err := b.take(small, 2<<20); err != nil {+ t.Fatal(err)+ }+ got := taking(b, big, 200<<20)+ waitQueued(t, b, 1)+ endSmall()+ if err := <-got; err != nil {+ t.Fatal(err)+ }+ if h := roomHeld(b); h != 200<<20 {+ t.Fatalf("held = %d MiB, want 200", h>>20)+ }+}++// First come, first served: a small body that came after a waiting big one+// waits behind it, so a big body isn't passed by a stream of small ones.+func TestBodyRoomKeepsOrder(t *testing.T) {+ b := roomOf(64 << 20)+ first, endFirst := context.WithCancel(context.Background())+ big, endBig := context.WithCancel(context.Background())+ later, endLater := context.WithCancel(context.Background())+ defer endBig()+ defer endLater()+ if err := b.take(first, 40<<20); err != nil {+ t.Fatal(err)+ }+ bigGot := taking(b, big, 40<<20)+ waitQueued(t, b, 1)+ laterGot := taking(b, later, 2<<20)+ waitQueued(t, b, 2)+ select {+ case <-laterGot:+ t.Fatal("a 2 MiB body went ahead of the 40 MiB one waiting before it")+ default:+ }+ endFirst()+ if err := <-bigGot; err != nil {+ t.Fatal(err)+ }+ if err := <-laterGot; err != nil {+ t.Fatal(err)+ }+}++// A body that waits bodyRoomWait is refused with what is held and why, and+// leaves the queue, so the one behind it is let in when room comes.+func TestBodyRoomRefusesAfterTheWait(t *testing.T) {+ was := bodyRoomWait+ bodyRoomWait = 50 * time.Millisecond+ defer func() { bodyRoomWait = was }()+ b := roomOf(64 << 20)+ ctx1, end1 := context.WithCancel(context.Background())+ ctx2, end2 := context.WithCancel(context.Background())+ defer end1()+ defer end2()+ if err := b.take(ctx1, 60<<20); err != nil {+ t.Fatal(err)+ }+ err := b.take(ctx2, 30<<20)+ var full *bodyFull+ if !errors.As(err, &full) || !strings.Contains(err.Error(), "60 MiB") || !strings.Contains(err.Error(), "the provider was not asked") {+ t.Fatalf("err = %v", err)+ }+ b.mu.Lock()+ q := len(b.queue)+ b.mu.Unlock()+ if q != 0 || roomHeld(b) != 60<<20 {+ t.Fatalf("queue = %d, held = %d MiB after a refusal", q, roomHeld(b)>>20)+ }+}++// A waiter whose client went away leaves the queue and holds nothing.+func TestBodyRoomGoneWaiterLeaves(t *testing.T) {+ b := roomOf(64 << 20)+ ctx1, end1 := context.WithCancel(context.Background())+ ctx2, end2 := context.WithCancel(context.Background())+ defer end1()+ if err := b.take(ctx1, 60<<20); err != nil {+ t.Fatal(err)+ }+ got := taking(b, ctx2, 30<<20)+ waitQueued(t, b, 1)+ end2()+ if err := <-got; !errors.Is(err, context.Canceled) {+ t.Fatalf("err = %v", err)+ }+ if h := roomHeld(b); h != 60<<20 {+ t.Fatalf("held = %d MiB", h>>20)+ }+}++// Small bodies, a context never done, a request reading its body twice+// and a machine whose memory isn't known take no room.+func TestBodyRoomTakesNothingItCantGiveBack(t *testing.T) {+ b := roomOf(64 << 20)+ ctx, end := context.WithCancel(context.Background())+ defer end()+ if err := b.take(ctx, bodyRoomFree-1); err != nil || roomHeld(b) != 0 {+ t.Fatalf("a small body: %v, held %d", err, roomHeld(b))+ }+ if err := b.take(context.Background(), 100<<20); err != nil || roomHeld(b) != 0 {+ t.Fatalf("a context never done: %v, held %d", err, roomHeld(b))+ }+ if err := b.take(ctx, 40<<20); err != nil {+ t.Fatal(err)+ }+ if err := b.take(ctx, 40<<20); err != nil || roomHeld(b) != 40<<20 {+ t.Fatalf("one request's second read: %v, held %d MiB", err, roomHeld(b)>>20)+ }+ if err := b.take(ctx, 50<<20); err != nil || roomHeld(b) != 50<<20 {+ t.Fatalf("a body read bigger than its request said: %v, held %d MiB", err, roomHeld(b)>>20)+ }+ unknown := roomOf(0)+ if err := unknown.take(ctx, 400<<20); err != nil || roomHeld(unknown) != 0 {+ t.Fatalf("memory not known: %v, held %d", err, roomHeld(unknown))+ }+}++// The room is a thirty-second of the memory magpie may use, 64 MiB at+// least: jihuayu's 1,670 MiB server serves one ~45 MiB body at a time.+func TestBodyRoomSize(t *testing.T) {+ root := t.TempDir()+ writeMeminfo := func(kb string) {+ t.Helper()+ if err := os.MkdirAll(filepath.Join(root, "proc"), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(root, "proc/meminfo"), []byte("MemTotal: "+kb+" kB\n"), 0o644); err != nil {+ t.Fatal(err)+ }+ }+ wasGOOS, wasRoot := memGOOS, memRoot+ memGOOS, memRoot = "linux", root+ defer func() { memGOOS, memRoot = wasGOOS, wasRoot }()+ writeMeminfo("1710080") // 1,670 MiB+ if n, why := bodyRoomSize(); n != bodyRoomFloor || !strings.Contains(why, "1670 MiB") {+ t.Fatalf("1,670 MiB: %d MiB (%s), want the floor", n>>20, why)+ }+ writeMeminfo("8388608") // 8 GiB+ if n, _ := bodyRoomSize(); n != 256<<20 {+ t.Fatalf("8 GiB: %d MiB, want 256", n>>20)+ }+}++// Through the gateway: a request whose body finds no room is answered 503+// in its protocol's envelope, and the provider is not asked.+func TestBodyRoomFullAnswers503(t *testing.T) {+ was, wasWait := bodyRoom, bodyRoomWait+ bodyRoom, bodyRoomWait = roomOf(64<<20), 50*time.Millisecond+ defer func() { bodyRoom, bodyRoomWait = was, wasWait }()+ hold, end := context.WithCancel(context.Background())+ defer end()+ if err := bodyRoom.take(hold, 60<<20); err != nil {+ t.Fatal(err)+ }+ s := &Server{}+ body := `{"model":"m","input":"` + strings.Repeat("x", 30<<20) + `"}`+ r := httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader([]byte(body)))+ ctx, cancel := context.WithCancel(r.Context())+ defer cancel()+ r = r.WithContext(ctx)+ w := httptest.NewRecorder()+ if got, ok := s.requestBody(w, r, provider.Responses); ok || got != nil {+ t.Fatal("a body with no room was read through")+ }+ if w.Code != http.StatusServiceUnavailable || !strings.Contains(w.Body.String(), "the provider was not asked") {+ t.Fatalf("%d %s", w.Code, w.Body.String())+ }+}diff --git a/internal/gateway/request_bounds.go b/internal/gateway/request_bounds.goindex 16d836f73..d0db1bf13 100644--- a/internal/gateway/request_bounds.go+++ b/internal/gateway/request_bounds.go@@ -44,11 +44,27 @@ func (s *Server) readRequestBody(w http.ResponseWriter, r *http.Request, from prif limit > 0 {limits.body = limit}+ // wait for room before reading: a body waiting is not held (bodyRoom)+ most := limits.body+ if most <= 0 {+ most = defaultBodyLimit+ }+ if n := r.ContentLength; n > 0 && n <= most {+ if err := bodyRoom.take(r.Context(), n); err != nil {+ w.Header().Set("Connection", "close") // its body is left unread+ writeError(w, from, http.StatusServiceUnavailable, err.Error())+ return nil, false+ }+ }body, status, err := readBoundedRequestBody(w, r, limits)if err != nil {writeError(w, from, status, err.Error())return nil, false}+ if err := bodyRoom.take(r.Context(), int64(len(body))); err != nil {+ writeError(w, from, http.StatusServiceUnavailable, err.Error())+ return nil, false+ }holdBody(r.Context(), body)return body, true}