Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a request body is read once into a slice of its declared length, not grown and copied (yg6888 on X: ~1 GB)

yetone2481750d926c
diff --git a/internal/gateway/request_bounds.go b/internal/gateway/request_bounds.go
index d165aa6d4..16d836f73 100644
--- a/internal/gateway/request_bounds.go
+++ b/internal/gateway/request_bounds.go
@@ -171,6 +171,11 @@ func readBoundedRequestBody(w http.ResponseWriter, r *http.Request, limits reque
}
return reject(http.StatusBadRequest, err)
}
+ // the wire length is the body's length only when it is sent as it is
+ declared := r.ContentLength
+ if enc := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Encoding"))); enc != "" && enc != "identity" {
+ declared = -1
+ }
original := r.Body
r.Body = &idleBody{original, controller, limits.readTimeout}
defer func() { r.Body = original }()
@@ -182,8 +187,8 @@ func readBoundedRequestBody(w http.ResponseWriter, r *http.Request, limits reque
return readFailure(err)
}
defer rd.Close()
- // Limit decoded bytes as well, without allocating from Content-Length.
- body, err := io.ReadAll(io.LimitReader(rd, limits.body+1))
+ // Limit decoded bytes as well.
+ body, err := readSized(io.LimitReader(rd, limits.body+1), declared)
if int64(len(body)) > limits.body {
return reject(http.StatusRequestEntityTooLarge, overLimit(limits.body))
}
@@ -193,6 +198,56 @@ func readBoundedRequestBody(w http.ResponseWriter, r *http.Request, limits reque
return body, 0, nil
}
+// sizedReadFirst is the most readSized sets aside before a byte has come:
+// a client that says more than it sends makes magpie hold no more than
+// this, or twice what it sent.
+const sizedReadFirst = 1 << 20
+
+// readSized reads r to its end into a slice the size the client declared,
+// when it declared one (declared > 0): io.ReadAll reads into chunks and
+// copies them into a slice of their sum, two copies of every body, and an
+// agent sends its whole conversation, ~1 MB, on every turn (yg6888 on X:
+// ~1 GB). Content-Length is not trusted for the allocation: room grows by
+// doubling, from sizedReadFirst, up to what was declared, so an honest body
+// of 1 MB or less is read into one slice of its own size, a larger one into
+// a final slice of its own size, and a body shorter than it said costs at
+// most about twice what it sent. A body longer than it said reads on, as
+// io.ReadAll would. The error is io.ReadAll's: nil at io.EOF.
+func readSized(r io.Reader, declared int64) ([]byte, error) {
+ if declared <= 0 {
+ return io.ReadAll(r)
+ }
+ b := make([]byte, 0, min(declared, sizedReadFirst))
+ for {
+ if len(b) == cap(b) {
+ if int64(len(b)) >= declared {
+ // all it said has come: look for its end before growing
+ var probe [512]byte
+ n, err := r.Read(probe[:])
+ if n > 0 {
+ b = append(b, probe[:n]...)
+ }
+ if err == io.EOF {
+ return b, nil
+ }
+ if err != nil {
+ return b, err
+ }
+ continue
+ }
+ b = append(make([]byte, 0, min(int64(cap(b))*2, declared)), b...)
+ }
+ n, err := r.Read(b[len(b):cap(b)])
+ b = b[:len(b)+n]
+ if err == io.EOF {
+ return b, nil
+ }
+ if err != nil {
+ return b, err
+ }
+ }
+}
+
// overLimit says a body is over magpie's own limit, and that it is
// magpie's, not the provider's: an agent shows the message as it comes,
// and a bare "Payload Too Large" read as the model's or the vendor's
diff --git a/internal/gateway/request_sized_test.go b/internal/gateway/request_sized_test.go
new file mode 100644
index 000000000..48887d13f
--- /dev/null
+++ b/internal/gateway/request_sized_test.go
@@ -0,0 +1,129 @@
+package gateway
+
+import (
+ "bytes"
+ "compress/gzip"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "runtime"
+ "strings"
+ "testing"
+
+ "github.com/yetone/magpie/internal/provider"
+)
+
+// chunked hands its bytes out a piece at a time, as a socket does.
+type chunked struct {
+ b []byte
+ size int
+}
+
+func (c *chunked) Read(p []byte) (int, error) {
+ if len(c.b) == 0 {
+ return 0, io.EOF
+ }
+ n := copy(p[:min(len(p), c.size)], c.b)
+ c.b = c.b[n:]
+ return n, nil
+}
+
+// allocated is the bytes f allocates.
+func allocated(f func()) uint64 {
+ var before, after runtime.MemStats
+ runtime.GC()
+ runtime.ReadMemStats(&before)
+ f()
+ runtime.ReadMemStats(&after)
+ return after.TotalAlloc - before.TotalAlloc
+}
+
+// An agent's whole conversation, sent with its length, is read into one
+// slice of its own size: io.ReadAll read it into chunks and copied them
+// into a slice of their sum, twice the body on every turn (yg6888 on X:
+// ~1 GB).
+func TestRequestBodyReadOnceAtItsLength(t *testing.T) {
+ sent := bytes.Repeat([]byte("0123456789abcdef"), 50_000) // 800 KB
+ var got []byte
+ var err error
+ once := allocated(func() { got, err = readSized(&chunked{sent, 32 << 10}, int64(len(sent))) })
+ if err != nil || !bytes.Equal(got, sent) {
+ t.Fatalf("read %d bytes, %v", len(got), err)
+ }
+ if cap(got) != len(got) {
+ t.Fatalf("cap %d for a body of %d", cap(got), len(got))
+ }
+ if limit := uint64(len(sent)) * 11 / 10; once > limit {
+ t.Fatalf("an %d-byte body allocated %d bytes to read", len(sent), once)
+ }
+ // the measure tells the two apart
+ if all := allocated(func() { io.ReadAll(&chunked{sent, 32 << 10}) }); all < uint64(len(sent))*3/2 {
+ t.Fatalf("io.ReadAll allocated only %d for %d bytes: the check above shows nothing", all, len(sent))
+ }
+
+ // larger than the first slice: its final slice is still its own size
+ big := bytes.Repeat(sent, 4)
+ got, err = readSized(&chunked{big, 64 << 10}, int64(len(big)))
+ if err != nil || !bytes.Equal(got, big) || cap(got) != len(got) {
+ t.Fatalf("a %d-byte body read as %d (cap %d), %v", len(big), len(got), cap(got), err)
+ }
+}
+
+// Content-Length is the client's word: a body shorter than it said costs
+// no more than the first slice, one longer reads on, a failed read is
+// returned as io.ReadAll returns it, and no length is io.ReadAll.
+func TestRequestBodyLengthNotTrusted(t *testing.T) {
+ var got []byte
+ lied := allocated(func() { got, _ = readSized(strings.NewReader("short"), defaultBodyLimit) })
+ if string(got) != "short" {
+ t.Fatalf("read %q", got)
+ }
+ if lied > sizedReadFirst+64<<10 {
+ t.Fatalf("a client that said 512 MB and sent 5 bytes made magpie allocate %d bytes", lied)
+ }
+ long := strings.Repeat("y", 5000)
+ if got, err := readSized(&chunked{[]byte(long), 700}, 10); err != nil || string(got) != long {
+ t.Fatalf("a body longer than it said read as %d bytes, %v", len(got), err)
+ }
+ if got, err := readSized(io.MultiReader(strings.NewReader("abc"), failedBodyReader{}), 100); !errors.Is(err, io.ErrUnexpectedEOF) || string(got) != "abc" {
+ t.Fatalf("failed read: %q, %v", got, err)
+ }
+ if got, err := readSized(strings.NewReader("abc"), -1); err != nil || string(got) != "abc" {
+ t.Fatalf("no length: %q, %v", got, err)
+ }
+}
+
+// Through the server: a body sent as it is is read at its length; an
+// encoded one, whose Content-Length is the wire's, is decoded whole.
+func TestRequestBodyReadAtItsLengthServed(t *testing.T) {
+ sent := strings.Repeat(`{"role":"user","content":"hi"},`, 20_000)
+ var got []byte
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ got, _ = New().requestBody(w, r, provider.Chat)
+ }))
+ defer srv.Close()
+ resp, err := http.Post(srv.URL, "application/json", strings.NewReader(sent))
+ if err != nil {
+ t.Fatal(err)
+ }
+ resp.Body.Close()
+ if string(got) != sent || cap(got) != len(got) {
+ t.Fatalf("read %d bytes (cap %d) of %d", len(got), cap(got), len(sent))
+ }
+
+ var gz bytes.Buffer
+ zw := gzip.NewWriter(&gz)
+ zw.Write([]byte(sent))
+ zw.Close()
+ req, _ := http.NewRequest("POST", srv.URL, &gz)
+ req.Header.Set("Content-Encoding", "gzip")
+ resp, err = http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resp.Body.Close()
+ if string(got) != sent {
+ t.Fatalf("a gzip body of %d bytes read as %d", len(sent), len(got))
+ }
+}