Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a zstd request body is decoded by a decoder kept between requests
yetone077c4720533e
diff --git a/internal/gateway/race_off_test.go b/internal/gateway/race_off_test.gonew file mode 100644index 000000000..a5fbbb347--- /dev/null+++ b/internal/gateway/race_off_test.go@@ -0,0 +1,5 @@+//go:build !race++package gateway++const raceOn = falsediff --git a/internal/gateway/race_on_test.go b/internal/gateway/race_on_test.gonew file mode 100644index 000000000..efeac588e--- /dev/null+++ b/internal/gateway/race_on_test.go@@ -0,0 +1,7 @@+//go:build race++package gateway++// raceOn says the tests run under -race, where sync.Pool lets a quarter of+// what is put in it go, so a test of what a pool saves is skipped.+const raceOn = truediff --git a/internal/gateway/request_bounds.go b/internal/gateway/request_bounds.goindex 92a48048f..d165aa6d4 100644--- a/internal/gateway/request_bounds.go+++ b/internal/gateway/request_bounds.go@@ -8,6 +8,7 @@ import ("net""net/http""strings"+ "sync""time""github.com/klauspost/compress/zstd"@@ -82,11 +83,13 @@ func decodedBody(r *http.Request) (io.ReadCloser, error) {switch enc := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Encoding"))); enc {case "", "identity":case "zstd":- d, err := zstd.NewReader(r.Body, zstd.WithDecoderMaxMemory(defaultBodyLimit))- if err != nil {+ d := bodyDecoders.Get().(*zstd.Decoder)+ if err := d.Reset(r.Body); err != nil {+ d.Reset(nil)+ bodyDecoders.Put(d)return nil, err}- rd = d.IOReadCloser()+ rd = &pooledZstd{d}case "gzip", "x-gzip":g, err := gzip.NewReader(r.Body)if err != nil {@@ -100,6 +103,40 @@ func decodedBody(r *http.Request) (io.ReadCloser, error) {return rd, nil}+// bodyDecoders keeps zstd decoders between requests' bodies. One made for+// each body (concurrent, the package's default) allocated its window and+// block buffers again every time: 1.7 GB of 9.8 GB allocated in 10 minutes+// of three Codex sessions' turns (procmem heap profile). One decodes in+// step with its reader, as the body is read through to its end anyway, and+// is let go with the pool's other idle ones by the garbage collector.+var bodyDecoders = sync.Pool{New: func() any {+ d, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(defaultBodyLimit))+ if err != nil {+ panic(err) // only for options it doesn't know+ }+ return d+}}++// pooledZstd is a body read through a pooled decoder, which goes back to+// the pool at Close; a read after it can't reach the decoder.+type pooledZstd struct{ d *zstd.Decoder }++func (z *pooledZstd) Read(p []byte) (int, error) {+ if z.d == nil {+ return 0, net.ErrClosed+ }+ return z.d.Read(p)+}++func (z *pooledZstd) Close() error {+ if z.d != nil {+ z.d.Reset(nil)+ bodyDecoders.Put(z.d)+ z.d = nil+ }+ return nil+}+// Read one bounded body, decoded; callers choose their protocol's error// envelope. The limit holds for the decoded bytes too.func readBoundedRequestBody(w http.ResponseWriter, r *http.Request, limits requestLimits) ([]byte, int, error) {diff --git a/internal/gateway/request_zstd_pool_test.go b/internal/gateway/request_zstd_pool_test.gonew file mode 100644index 000000000..23224acc3--- /dev/null+++ b/internal/gateway/request_zstd_pool_test.go@@ -0,0 +1,50 @@+package gateway++import (+ "bytes"+ "net/http/httptest"+ "runtime"+ "strings"+ "testing"++ "github.com/klauspost/compress/zstd"+)++// A zstd body is read through a decoder kept between requests: one made+// for each body allocated its window and block buffers every time, 1.7 GB+// of 9.8 GB in 10 minutes of three Codex sessions (procmem heap profile).+// What a read allocates past the decoded body itself stays small.+func TestZstdBodyDecoderKept(t *testing.T) {+ plain := []byte(`{"model":"fake/m1","input":"` + strings.Repeat("please look at file 12 and fix the bug ", 8000) + `"}`)+ enc, _ := zstd.NewWriter(nil)+ packed := enc.EncodeAll(plain, nil)+ enc.Close()+ read := func() {+ t.Helper()+ r := httptest.NewRequest("POST", "/v1/responses", bytes.NewReader(packed))+ r.Header.Set("Content-Encoding", "zstd")+ body, status, err := readBoundedRequestBody(httptest.NewRecorder(), r, requestLimits{})+ if err != nil || status != 0 || !bytes.Equal(body, plain) {+ t.Fatalf("read %d bytes, status %d, %v", len(body), status, err)+ }+ }+ read() // the pool's first decoder+ if raceOn {+ t.Skip("-race: sync.Pool lets a quarter of what is put in it go")+ }+ const n = 20+ var before, after runtime.MemStats+ runtime.GC()+ runtime.ReadMemStats(&before)+ for range n {+ read()+ }+ runtime.ReadMemStats(&after)+ // io.ReadAll's growth to the body's size is ~2-3x it; a decoder made+ // for each body added its window and buffers on top+ per := (after.TotalAlloc - before.TotalAlloc) / n+ if per > uint64(4*len(plain)) {+ t.Fatalf("a %d KB body allocated %d KB to read", len(plain)>>10, per>>10)+ }+ t.Logf("a %d KB body: %d KB a read", len(plain)>>10, per>>10)+}