Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a request body is read once into a slice of its declared length, not grown and copied (yg6888 on X: ~1 GB)
yetone2481750d926c
diff --git a/internal/gateway/request_bounds.go b/internal/gateway/request_bounds.goindex d165aa6d4..16d836f73 100644--- a/internal/gateway/request_bounds.go+++ b/internal/gateway/request_bounds.go@@ -171,6 +171,11 @@ func readBoundedRequestBody(w http.ResponseWriter, r *http.Request, limits reque}return reject(http.StatusBadRequest, err)}+ // the wire length is the body's length only when it is sent as it is+ declared := r.ContentLength+ if enc := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Encoding"))); enc != "" && enc != "identity" {+ declared = -1+ }original := r.Bodyr.Body = &idleBody{original, controller, limits.readTimeout}defer func() { r.Body = original }()@@ -182,8 +187,8 @@ func readBoundedRequestBody(w http.ResponseWriter, r *http.Request, limits requereturn readFailure(err)}defer rd.Close()- // Limit decoded bytes as well, without allocating from Content-Length.- body, err := io.ReadAll(io.LimitReader(rd, limits.body+1))+ // Limit decoded bytes as well.+ body, err := readSized(io.LimitReader(rd, limits.body+1), declared)if int64(len(body)) > limits.body {return reject(http.StatusRequestEntityTooLarge, overLimit(limits.body))}@@ -193,6 +198,56 @@ func readBoundedRequestBody(w http.ResponseWriter, r *http.Request, limits requereturn body, 0, nil}+// sizedReadFirst is the most readSized sets aside before a byte has come:+// a client that says more than it sends makes magpie hold no more than+// this, or twice what it sent.+const sizedReadFirst = 1 << 20++// readSized reads r to its end into a slice the size the client declared,+// when it declared one (declared > 0): io.ReadAll reads into chunks and+// copies them into a slice of their sum, two copies of every body, and an+// agent sends its whole conversation, ~1 MB, on every turn (yg6888 on X:+// ~1 GB). Content-Length is not trusted for the allocation: room grows by+// doubling, from sizedReadFirst, up to what was declared, so an honest body+// of 1 MB or less is read into one slice of its own size, a larger one into+// a final slice of its own size, and a body shorter than it said costs at+// most about twice what it sent. A body longer than it said reads on, as+// io.ReadAll would. The error is io.ReadAll's: nil at io.EOF.+func readSized(r io.Reader, declared int64) ([]byte, error) {+ if declared <= 0 {+ return io.ReadAll(r)+ }+ b := make([]byte, 0, min(declared, sizedReadFirst))+ for {+ if len(b) == cap(b) {+ if int64(len(b)) >= declared {+ // all it said has come: look for its end before growing+ var probe [512]byte+ n, err := r.Read(probe[:])+ if n > 0 {+ b = append(b, probe[:n]...)+ }+ if err == io.EOF {+ return b, nil+ }+ if err != nil {+ return b, err+ }+ continue+ }+ b = append(make([]byte, 0, min(int64(cap(b))*2, declared)), b...)+ }+ n, err := r.Read(b[len(b):cap(b)])+ b = b[:len(b)+n]+ if err == io.EOF {+ return b, nil+ }+ if err != nil {+ return b, err+ }+ }+}+// overLimit says a body is over magpie's own limit, and that it is// magpie's, not the provider's: an agent shows the message as it comes,// and a bare "Payload Too Large" read as the model's or the vendor'sdiff --git a/internal/gateway/request_sized_test.go b/internal/gateway/request_sized_test.gonew file mode 100644index 000000000..48887d13f--- /dev/null+++ b/internal/gateway/request_sized_test.go@@ -0,0 +1,129 @@+package gateway++import (+ "bytes"+ "compress/gzip"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "runtime"+ "strings"+ "testing"++ "github.com/yetone/magpie/internal/provider"+)++// chunked hands its bytes out a piece at a time, as a socket does.+type chunked struct {+ b []byte+ size int+}++func (c *chunked) Read(p []byte) (int, error) {+ if len(c.b) == 0 {+ return 0, io.EOF+ }+ n := copy(p[:min(len(p), c.size)], c.b)+ c.b = c.b[n:]+ return n, nil+}++// allocated is the bytes f allocates.+func allocated(f func()) uint64 {+ var before, after runtime.MemStats+ runtime.GC()+ runtime.ReadMemStats(&before)+ f()+ runtime.ReadMemStats(&after)+ return after.TotalAlloc - before.TotalAlloc+}++// An agent's whole conversation, sent with its length, is read into one+// slice of its own size: io.ReadAll read it into chunks and copied them+// into a slice of their sum, twice the body on every turn (yg6888 on X:+// ~1 GB).+func TestRequestBodyReadOnceAtItsLength(t *testing.T) {+ sent := bytes.Repeat([]byte("0123456789abcdef"), 50_000) // 800 KB+ var got []byte+ var err error+ once := allocated(func() { got, err = readSized(&chunked{sent, 32 << 10}, int64(len(sent))) })+ if err != nil || !bytes.Equal(got, sent) {+ t.Fatalf("read %d bytes, %v", len(got), err)+ }+ if cap(got) != len(got) {+ t.Fatalf("cap %d for a body of %d", cap(got), len(got))+ }+ if limit := uint64(len(sent)) * 11 / 10; once > limit {+ t.Fatalf("an %d-byte body allocated %d bytes to read", len(sent), once)+ }+ // the measure tells the two apart+ if all := allocated(func() { io.ReadAll(&chunked{sent, 32 << 10}) }); all < uint64(len(sent))*3/2 {+ t.Fatalf("io.ReadAll allocated only %d for %d bytes: the check above shows nothing", all, len(sent))+ }++ // larger than the first slice: its final slice is still its own size+ big := bytes.Repeat(sent, 4)+ got, err = readSized(&chunked{big, 64 << 10}, int64(len(big)))+ if err != nil || !bytes.Equal(got, big) || cap(got) != len(got) {+ t.Fatalf("a %d-byte body read as %d (cap %d), %v", len(big), len(got), cap(got), err)+ }+}++// Content-Length is the client's word: a body shorter than it said costs+// no more than the first slice, one longer reads on, a failed read is+// returned as io.ReadAll returns it, and no length is io.ReadAll.+func TestRequestBodyLengthNotTrusted(t *testing.T) {+ var got []byte+ lied := allocated(func() { got, _ = readSized(strings.NewReader("short"), defaultBodyLimit) })+ if string(got) != "short" {+ t.Fatalf("read %q", got)+ }+ if lied > sizedReadFirst+64<<10 {+ t.Fatalf("a client that said 512 MB and sent 5 bytes made magpie allocate %d bytes", lied)+ }+ long := strings.Repeat("y", 5000)+ if got, err := readSized(&chunked{[]byte(long), 700}, 10); err != nil || string(got) != long {+ t.Fatalf("a body longer than it said read as %d bytes, %v", len(got), err)+ }+ if got, err := readSized(io.MultiReader(strings.NewReader("abc"), failedBodyReader{}), 100); !errors.Is(err, io.ErrUnexpectedEOF) || string(got) != "abc" {+ t.Fatalf("failed read: %q, %v", got, err)+ }+ if got, err := readSized(strings.NewReader("abc"), -1); err != nil || string(got) != "abc" {+ t.Fatalf("no length: %q, %v", got, err)+ }+}++// Through the server: a body sent as it is is read at its length; an+// encoded one, whose Content-Length is the wire's, is decoded whole.+func TestRequestBodyReadAtItsLengthServed(t *testing.T) {+ sent := strings.Repeat(`{"role":"user","content":"hi"},`, 20_000)+ var got []byte+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ got, _ = New().requestBody(w, r, provider.Chat)+ }))+ defer srv.Close()+ resp, err := http.Post(srv.URL, "application/json", strings.NewReader(sent))+ if err != nil {+ t.Fatal(err)+ }+ resp.Body.Close()+ if string(got) != sent || cap(got) != len(got) {+ t.Fatalf("read %d bytes (cap %d) of %d", len(got), cap(got), len(sent))+ }++ var gz bytes.Buffer+ zw := gzip.NewWriter(&gz)+ zw.Write([]byte(sent))+ zw.Close()+ req, _ := http.NewRequest("POST", srv.URL, &gz)+ req.Header.Set("Content-Encoding", "gzip")+ resp, err = http.DefaultClient.Do(req)+ if err != nil {+ t.Fatal(err)+ }+ resp.Body.Close()+ if string(got) != sent {+ t.Fatalf("a gzip body of %d bytes read as %d", len(sent), len(got))+ }+}