Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a codex image's header can't make its decode hold what it declares

zzjcool3eee19b08be2
diff --git a/internal/gateway/codex_images.go b/internal/gateway/codex_images.go
index e7c2e844b..abf573319 100644
--- a/internal/gateway/codex_images.go
+++ b/internal/gateway/codex_images.go
@@ -7,6 +7,7 @@ import (
"encoding/base64"
"encoding/binary"
"image"
+ "image/color"
"image/draw"
"image/gif"
"image/jpeg"
@@ -134,6 +135,15 @@ type fitJob struct {
// Retina one.
const fitWorkers = 4
+// fitBudget is how many bytes of decoded image one request may hold at
+// once, across fitWorkers of them. fitWorkers alone bounds nothing: four
+// images made side by side, each up to what one image may hold, is 4
+// times codexImageMaxBytes, and a request that asks for that much is a
+// process the system kills (yg6888 on Discord: a few agents at once put
+// the heap past 300 MB with 1.5 MB bodies). A request's images are made in
+// batches that fit here together, so the second batch waits for the first.
+const fitBudget = 512 << 20
+
// fitAll fits each job's image: as it was kept (fitted), or else made
// again, the images not kept made side by side.
func fitAll(jobs []fitJob) {
@@ -164,9 +174,53 @@ func fitAll(jobs []fitJob) {
order = append(order, i)
}
}
+ // what each will hold once made, from its header, so that the batches
+ // below hold no more than fitBudget between them
+ holds := make([]int, len(order))
+ for n, i := range order {
+ holds[n] = fitHolds(jobs[i].url)
+ }
+ for _, b := range fitInHand(holds, fitBudget) {
+ fitBatch(jobs, order[b[0]:b[1]], made)
+ }
+ for _, i := range order {
+ fitted.put(jobs[i].key, *made[jobs[i].key])
+ }
+ for _, i := range todo {
+ e := made[jobs[i].key]
+ jobs[i].with, jobs[i].changed = e.url(), e.data != nil
+ }
+}
+
+// fitInHand splits what is to be made into the batches made one after
+// another: no batch holds more than budget once its images are made,
+// except one image that holds more than the budget by itself, which is
+// made alone and turned away by codexImageRefit. Each batch is a pair of
+// where it starts and where it ends in holds.
+func fitInHand(holds []int, budget int) [][2]int {
+ var out [][2]int
+ for at := 0; at < len(holds); {
+ room, end := budget, at
+ for ; end < len(holds); end++ {
+ if holds[end] > room {
+ break
+ }
+ room -= holds[end]
+ }
+ if end == at {
+ end++
+ }
+ out = append(out, [2]int{at, end})
+ at = end
+ }
+ return out
+}
+
+// fitBatch makes the images of a batch, fitWorkers of them at once.
+func fitBatch(jobs []fitJob, batch []int, made map[[32]byte]*fitEntry) {
next := make(chan int)
var wg sync.WaitGroup
- for range min(fitWorkers, runtime.GOMAXPROCS(0), len(order)) {
+ for range min(fitWorkers, runtime.GOMAXPROCS(0), len(batch)) {
wg.Add(1)
go func() {
defer wg.Done()
@@ -176,18 +230,11 @@ func fitAll(jobs []fitJob) {
}
}()
}
- for _, i := range order {
+ for _, i := range batch {
next <- i
}
close(next)
wg.Wait()
- for _, i := range order {
- fitted.put(jobs[i].key, *made[jobs[i].key])
- }
- for _, i := range todo {
- e := made[jobs[i].key]
- jobs[i].with, jobs[i].changed = e.url(), e.data != nil
- }
}
type imageLimits struct{ side, patches int }
@@ -232,9 +279,79 @@ func (l imageLimits) size(w, h int) (int, int) {
return max(int(math.Floor(fw*scale)), 1), max(int(math.Floor(fh*scale)), 1)
}
-// codexImageMaxPixels keeps a request from decoding an image too large
-// to hold: one past it goes as it came.
-const codexImageMaxPixels = 80 << 20
+// codexImageMaxBytes is what one image may hold once decoded, its pixels
+// and the bytes they came in: past it the image goes as it came.
+//
+// It used to be 80<<20 pixels, which is not what a decode holds. A header
+// of a few bytes says what size an image is and the decoder allocates what
+// it says before it reads a pixel, so 65 bytes declaring 8000x8000 at 16
+// bits a channel held 512 MB, and four of them in one request held 1.8 GB
+// (measured with runtime.MemStats). The bits a channel costs are the
+// format's, and imageHolds reads them from the header.
+const codexImageMaxBytes = 320 << 20
+
+// holdsMost is the most imageHolds tells: one past the budget, so that a
+// declared size no int holds still counts as too large.
+const holdsMost = codexImageMaxBytes + 1
+
+// imageHolds is what decoding an image of this header config holds: its
+// pixels in the image its format makes of them. That's the format's own
+// check above, so that the bytes a request holds follow the size it
+// declares and the bits each channel costs, not the pixels alone.
+func imageHolds(cfg image.Config, format string) (int, bool) {
+ if cfg.Width <= 0 || cfg.Height <= 0 {
+ return 0, false
+ }
+ // 4 bytes a pixel for RGBA, which anything with colour or alpha is
+ // decoded into; 1 for grey; 2 for grey with 16 bits a channel and 8
+ // for colour with 16, as image/png's readImagePass allocates them.
+ bytesPerPixel := 4
+ switch {
+ case cfg.ColorModel == color.GrayModel:
+ bytesPerPixel = 1
+ case format == "png" && cfg.ColorModel == color.Gray16Model:
+ bytesPerPixel = 2
+ case format == "png" && (cfg.ColorModel == color.RGBA64Model || cfg.ColorModel == color.NRGBA64Model):
+ bytesPerPixel = 8
+ }
+ pixels := int64(cfg.Width) * int64(cfg.Height)
+ if pixels <= 0 || pixels*int64(bytesPerPixel) > holdsMost {
+ return holdsMost, true
+ }
+ return int(pixels) * bytesPerPixel, true
+}
+
+// fitsHead is how much of an image's base64 fitHolds reads to tell its
+// header: enough for a PNG's chunks and a JPEG's segments, and little
+// enough that asking costs nothing.
+const fitsHead = 4 << 10
+
+// fitHolds is what making an image of url will hold, from its header
+// alone, so that nothing is decoded to find out. An image it can't read
+// holds nothing and isn't made.
+func fitHolds(url string) int {
+ comma := strings.IndexByte(url, ',')
+ if comma < 0 || !strings.HasSuffix(url[:comma], ";base64") {
+ return 0
+ }
+ b64 := url[comma+1:]
+ if len(b64) > fitsHead {
+ b64 = b64[:fitsHead&^3] // whole groups of four, as base64 reads them
+ }
+ head, err := base64.StdEncoding.DecodeString(b64)
+ if err != nil {
+ return 0
+ }
+ cfg, format, err := image.DecodeConfig(bytes.NewReader(head))
+ if err != nil {
+ return 0
+ }
+ holds, ok := imageHolds(cfg, format)
+ if !ok {
+ return 0
+ }
+ return holds
+}
// fitEntry is what an image is sent as: data in mime, or, with no data,
// the image as it came.
@@ -347,12 +464,21 @@ var refits atomic.Int64
// codexImageRefit is an image's base64 resized and written again, or an
// entry with no data when it goes as it came.
func codexImageRefit(b64 string, l imageLimits) fitEntry {
+ if len(b64) > codexImageMaxBytes {
+ return fitEntry{} // more than it may hold before it is decoded
+ }
raw, err := base64.StdEncoding.DecodeString(b64)
if err != nil {
return fitEntry{}
}
cfg, format, err := image.DecodeConfig(bytes.NewReader(raw))
- if err != nil || cfg.Width*cfg.Height > codexImageMaxPixels {
+ if err != nil {
+ return fitEntry{}
+ }
+ // what the pixels and the bytes they came in will hold together, so
+ // that a header of a few bytes can't ask for a gigabyte
+ holds, ok := imageHolds(cfg, format)
+ if !ok || len(raw)+holds > codexImageMaxBytes {
return fitEntry{}
}
w, h := l.size(cfg.Width, cfg.Height)
diff --git a/internal/gateway/codex_images_test.go b/internal/gateway/codex_images_test.go
index 4c0e815cf..bdc5dc0a3 100644
--- a/internal/gateway/codex_images_test.go
+++ b/internal/gateway/codex_images_test.go
@@ -4,8 +4,10 @@ import (
"bytes"
"container/list"
"encoding/base64"
+ "encoding/binary"
"encoding/json"
"fmt"
+ "hash/crc32"
"image"
"image/color"
"image/jpeg"
@@ -14,6 +16,7 @@ import (
"math/rand/v2"
"net/http"
"net/http/httptest"
+ "runtime"
"strings"
"testing"
"time"
@@ -377,6 +380,215 @@ func fitKeyEntry(t *testing.T, shot string) fitEntry {
return e
}
+// pngHeader is a PNG whose header declares w by h at depth bits a channel
+// and colour, with a valid zlib stream of no pixels after it: what a
+// decode reads the size from, before it reads a pixel. Go's png decodes it
+// as far as allocating the image the header declares, which is where a
+// header alone costs its bytes.
+func pngHeader(w, h uint32, depth, colour byte) []byte {
+ var b bytes.Buffer
+ b.Write([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'})
+ be := func(v any) { _ = binary.Write(&b, binary.BigEndian, v) }
+ var ihdr bytes.Buffer
+ _ = binary.Write(&ihdr, binary.BigEndian, w)
+ _ = binary.Write(&ihdr, binary.BigEndian, h)
+ ihdr.Write([]byte{depth, colour, 0, 0, 0})
+ for _, c := range []struct {
+ kind string
+ body []byte
+ }{
+ {"IHDR", ihdr.Bytes()},
+ // a valid zlib stream of no pixel data, so that a decode reads
+ // the header and allocates what it says
+ {"IDAT", []byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01}},
+ {"IEND", nil},
+ } {
+ be(uint32(len(c.body)))
+ head := append([]byte(c.kind), c.body...)
+ b.Write(head)
+ be(crc32.ChecksumIEEE(head))
+ }
+ return b.Bytes()
+}
+
+func bombURL(w, h uint32, depth, colour byte) string {
+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(pngHeader(w, h, depth, colour))
+}
+
+// allocDuring is the bytes f allocates, from before it runs to after,
+// which is what a decode of a size it was given costs whether or not it
+// then finds the pixels.
+func allocDuring(f func()) uint64 {
+ runtime.GC()
+ var before, after runtime.MemStats
+ runtime.ReadMemStats(&before)
+ f()
+ runtime.ReadMemStats(&after)
+ return after.TotalAlloc - before.TotalAlloc
+}
+
+// A header of a few dozen bytes is all it takes to ask a decode for
+// hundreds of megabytes: it declares the size, and png allocates that
+// before it reads a pixel, so the decode is paid for even though it then
+// fails. 110 bytes declaring 8000x8000 at 16 bits a channel held 488 MB,
+// and four such images in one request held 1.7 GB: a process the system
+// kills, off 260 bytes of an upload. What a decode may hold is judged
+// from the declaration now, before it decodes, and it is the same measure
+// for every format (codexImageMaxBytes, four bytes a pixel for anything
+// with colour, as image/png allocates it).
+func TestCodexAHeaderAloneCantHoldPastTheBudget(t *testing.T) {
+ if testing.Short() {
+ t.Skip("measures the bytes a decode holds")
+ }
+ for _, c := range []struct {
+ w, h uint32
+ depth byte
+ colour byte
+ what string
+ }{
+ {8000, 8000, 8, 6, "8000x8000 at 8 bits a channel: 256 MB of pixels"},
+ {8000, 8000, 16, 6, "8000x8000 at 16 bits a channel: 512 MB of pixels"},
+ {70000, 70000, 8, 6, "70000x70000: more than an int holds"},
+ } {
+ url := bombURL(c.w, c.h, c.depth, c.colour)
+ if n := len(url); n > 1024 {
+ t.Fatalf("%s: the upload is %d bytes, not a header alone", c.what, n)
+ }
+ b64 := url[strings.IndexByte(url, ',')+1:]
+ held := allocDuring(func() {
+ codexImageRefit(b64, imageLimits{2048, 2500})
+ })
+ if held > codexImageMaxBytes {
+ t.Errorf("%s: %d bytes of upload held %d MB, past the %d MB a decode may hold",
+ c.what, len(url), held>>20, codexImageMaxBytes>>20)
+ }
+ }
+}
+
+// The same, in one request: four headers each declaring a size it may not
+// hold, which codexImagesFit used to make side by side.
+func TestCodexFourHeadersInOneRequestCantHoldTheHeap(t *testing.T) {
+ const allowed = 128 << 20
+ if testing.Short() {
+ t.Skip("measures the bytes a decode holds")
+ }
+ // four *distinct* images: one entry is made per image, so the same data
+ // URL four times is made once
+ urls := make([]string, 4)
+ for i := range urls {
+ urls[i] = bombURL(7000+uint32(i), 8000, 16, 6)
+ }
+ body := computerUseRequest(urls)
+ held := allocDuring(func() {
+ codexImagesFit(codexAccount(), provider.Responses, body)
+ })
+ if held > allowed {
+ t.Errorf("%d bytes of upload, sent as 4 images in one request, held %d MB, past %d MB",
+ len(body), held>>20, allowed>>20)
+ }
+}
+
+// The budget the bytes above are counted against is the one a real
+// screenshot of the size Codex sends still fits under: what it holds once
+// decoded, so that a genuine image and a header alone are judged by the
+// same measure.
+func TestCodexRealScreenshotsFitBytesTheyHold(t *testing.T) {
+ for _, c := range []struct{ w, h int }{
+ {3456, 2234}, // ylorn's, the size Codex's own client sends
+ {4032, 3024},
+ {6000, 4000},
+ } {
+ for _, format := range []string{"png", "jpeg"} {
+ url := shotURL(t, shotImage(c.w, c.h), format)
+ raw := rawOf(t, url)
+ cfg, f, err := image.DecodeConfig(bytes.NewReader(raw))
+ if err != nil {
+ t.Fatal(err)
+ }
+ holds, ok := imageHolds(cfg, f)
+ if !ok || len(raw)+holds > codexImageMaxBytes {
+ t.Fatalf("%dx%d %s holds %d MB: a real screenshot is turned away", c.w, c.h, format, holds>>20)
+ }
+ e := codexImageRefit(url[strings.IndexByte(url, ',')+1:], imageLimits{2048, 2500})
+ if e.data == nil {
+ t.Fatalf("%dx%d %s went as it came", c.w, c.h, format)
+ }
+ if m, _ := shotDecode(t, "data:"+e.mime+";base64,"+base64.StdEncoding.EncodeToString(e.data)); m == nil {
+ t.Fatalf("%dx%d %s: what was made isn't an image", c.w, c.h, format)
+ }
+ }
+ }
+}
+
+func rawOf(t *testing.T, url string) []byte {
+ t.Helper()
+ raw, err := base64.StdEncoding.DecodeString(url[strings.IndexByte(url, ',')+1:])
+ if err != nil {
+ t.Fatal(err)
+ }
+ return raw
+}
+
+// What one request may hold is batched: no more than fitBudget is made at
+// once, however many images the request carries, so that a request of 100
+// screenshots holds what four do and not what a hundred do.
+func TestCodexFittingHoldsTheBudget(t *testing.T) {
+ const mb = 1 << 20
+ for _, c := range []struct {
+ what string
+ holds []int
+ budget int
+ want [][2]int
+ }{
+ {"four small ones, all at once", []int{10 * mb, 10 * mb, 10 * mb, 10 * mb}, 100 * mb, [][2]int{{0, 4}}},
+ {"one more than the budget, alone", []int{10 * mb, 250 * mb, 10 * mb}, 100 * mb, [][2]int{{0, 1}, {1, 2}, {2, 3}}},
+ {"the second waits for the first", []int{60 * mb, 60 * mb}, 100 * mb, [][2]int{{0, 1}, {1, 2}}},
+ {"a batch breaks where the next wouldn't fit", []int{40 * mb, 40 * mb, 40 * mb, 40 * mb, 10 * mb}, 100 * mb, [][2]int{{0, 2}, {2, 5}}},
+ {"a batch breaks where the next wouldn't fit, exactly", []int{40 * mb, 40 * mb, 40 * mb, 30 * mb}, 100 * mb, [][2]int{{0, 2}, {2, 4}}},
+ {"one past the budget on its own, alone", []int{400 * mb, 10 * mb}, 100 * mb, [][2]int{{0, 1}, {1, 2}}},
+ {"nothing to make", nil, 100 * mb, nil},
+ } {
+ got := fitInHand(c.holds, c.budget)
+ if len(got) != len(c.want) {
+ t.Errorf("%s: %d batches %v, want %d %v", c.what, len(got), got, len(c.want), c.want)
+ continue
+ }
+ for i := range got {
+ if got[i] != c.want[i] {
+ t.Errorf("%s: batch %d is %v, want %v", c.what, i, got[i], c.want[i])
+ }
+ }
+ // and what any batch holds is the budget, or one image alone
+ at := 0
+ for _, b := range got {
+ if b[0] != at {
+ t.Errorf("%s: batch %v leaves a gap after %d", c.what, b, at)
+ }
+ at = b[1]
+ sum := 0
+ for _, h := range c.holds[b[0]:b[1]] {
+ sum += h
+ }
+ if sum > c.budget && b[1]-b[0] != 1 {
+ t.Errorf("%s: batch %v holds %d MB past the budget of %d MB", c.what, b, sum/mb, c.budget/mb)
+ }
+ }
+ }
+}
+
+// A request whose images fit the budget is made as it was: the batching
+// changes nothing for the screenshots a real turn carries.
+func TestCodexFittingLeavesATurnThatFitsAsItWas(t *testing.T) {
+ made := freshFitted(t, fittedBudget)
+ shots := wideShots(t, 8)
+ if out := codexImagesFit(codexAccount(), provider.Responses, computerUseRequest(shots)); len(out) == 0 {
+ t.Fatal("no reply body")
+ }
+ if n := made(); n != 8 {
+ t.Errorf("%d of 8 images made", n)
+ }
+}
+
// What no request has sent for fittedIdle is let go.
func TestCodexFittedImagesLetGoWhenIdle(t *testing.T) {
made := freshFitted(t, fittedBudget)