Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a zstd request body is decoded by a decoder kept between requests

yetone077c4720533e
diff --git a/internal/gateway/race_off_test.go b/internal/gateway/race_off_test.go
new file mode 100644
index 000000000..a5fbbb347
--- /dev/null
+++ b/internal/gateway/race_off_test.go
@@ -0,0 +1,5 @@
+//go:build !race
+
+package gateway
+
+const raceOn = false
diff --git a/internal/gateway/race_on_test.go b/internal/gateway/race_on_test.go
new file mode 100644
index 000000000..efeac588e
--- /dev/null
+++ b/internal/gateway/race_on_test.go
@@ -0,0 +1,7 @@
+//go:build race
+
+package gateway
+
+// raceOn says the tests run under -race, where sync.Pool lets a quarter of
+// what is put in it go, so a test of what a pool saves is skipped.
+const raceOn = true
diff --git a/internal/gateway/request_bounds.go b/internal/gateway/request_bounds.go
index 92a48048f..d165aa6d4 100644
--- a/internal/gateway/request_bounds.go
+++ b/internal/gateway/request_bounds.go
@@ -8,6 +8,7 @@ import (
"net"
"net/http"
"strings"
+ "sync"
"time"
"github.com/klauspost/compress/zstd"
@@ -82,11 +83,13 @@ func decodedBody(r *http.Request) (io.ReadCloser, error) {
switch enc := strings.ToLower(strings.TrimSpace(r.Header.Get("Content-Encoding"))); enc {
case "", "identity":
case "zstd":
- d, err := zstd.NewReader(r.Body, zstd.WithDecoderMaxMemory(defaultBodyLimit))
- if err != nil {
+ d := bodyDecoders.Get().(*zstd.Decoder)
+ if err := d.Reset(r.Body); err != nil {
+ d.Reset(nil)
+ bodyDecoders.Put(d)
return nil, err
}
- rd = d.IOReadCloser()
+ rd = &pooledZstd{d}
case "gzip", "x-gzip":
g, err := gzip.NewReader(r.Body)
if err != nil {
@@ -100,6 +103,40 @@ func decodedBody(r *http.Request) (io.ReadCloser, error) {
return rd, nil
}
+// bodyDecoders keeps zstd decoders between requests' bodies. One made for
+// each body (concurrent, the package's default) allocated its window and
+// block buffers again every time: 1.7 GB of 9.8 GB allocated in 10 minutes
+// of three Codex sessions' turns (procmem heap profile). One decodes in
+// step with its reader, as the body is read through to its end anyway, and
+// is let go with the pool's other idle ones by the garbage collector.
+var bodyDecoders = sync.Pool{New: func() any {
+ d, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(1), zstd.WithDecoderMaxMemory(defaultBodyLimit))
+ if err != nil {
+ panic(err) // only for options it doesn't know
+ }
+ return d
+}}
+
+// pooledZstd is a body read through a pooled decoder, which goes back to
+// the pool at Close; a read after it can't reach the decoder.
+type pooledZstd struct{ d *zstd.Decoder }
+
+func (z *pooledZstd) Read(p []byte) (int, error) {
+ if z.d == nil {
+ return 0, net.ErrClosed
+ }
+ return z.d.Read(p)
+}
+
+func (z *pooledZstd) Close() error {
+ if z.d != nil {
+ z.d.Reset(nil)
+ bodyDecoders.Put(z.d)
+ z.d = nil
+ }
+ return nil
+}
+
// Read one bounded body, decoded; callers choose their protocol's error
// envelope. The limit holds for the decoded bytes too.
func readBoundedRequestBody(w http.ResponseWriter, r *http.Request, limits requestLimits) ([]byte, int, error) {
diff --git a/internal/gateway/request_zstd_pool_test.go b/internal/gateway/request_zstd_pool_test.go
new file mode 100644
index 000000000..23224acc3
--- /dev/null
+++ b/internal/gateway/request_zstd_pool_test.go
@@ -0,0 +1,50 @@
+package gateway
+
+import (
+ "bytes"
+ "net/http/httptest"
+ "runtime"
+ "strings"
+ "testing"
+
+ "github.com/klauspost/compress/zstd"
+)
+
+// A zstd body is read through a decoder kept between requests: one made
+// for each body allocated its window and block buffers every time, 1.7 GB
+// of 9.8 GB in 10 minutes of three Codex sessions (procmem heap profile).
+// What a read allocates past the decoded body itself stays small.
+func TestZstdBodyDecoderKept(t *testing.T) {
+ plain := []byte(`{"model":"fake/m1","input":"` + strings.Repeat("please look at file 12 and fix the bug ", 8000) + `"}`)
+ enc, _ := zstd.NewWriter(nil)
+ packed := enc.EncodeAll(plain, nil)
+ enc.Close()
+ read := func() {
+ t.Helper()
+ r := httptest.NewRequest("POST", "/v1/responses", bytes.NewReader(packed))
+ r.Header.Set("Content-Encoding", "zstd")
+ body, status, err := readBoundedRequestBody(httptest.NewRecorder(), r, requestLimits{})
+ if err != nil || status != 0 || !bytes.Equal(body, plain) {
+ t.Fatalf("read %d bytes, status %d, %v", len(body), status, err)
+ }
+ }
+ read() // the pool's first decoder
+ if raceOn {
+ t.Skip("-race: sync.Pool lets a quarter of what is put in it go")
+ }
+ const n = 20
+ var before, after runtime.MemStats
+ runtime.GC()
+ runtime.ReadMemStats(&before)
+ for range n {
+ read()
+ }
+ runtime.ReadMemStats(&after)
+ // io.ReadAll's growth to the body's size is ~2-3x it; a decoder made
+ // for each body added its window and buffers on top
+ per := (after.TotalAlloc - before.TotalAlloc) / n
+ if per > uint64(4*len(plain)) {
+ t.Fatalf("a %d KB body allocated %d KB to read", len(plain)>>10, per>>10)
+ }
+ t.Logf("a %d KB body: %d KB a read", len(plain)>>10, per>>10)
+}