Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a codex image's header can't make its decode hold what it declares
zzjcool3eee19b08be2
diff --git a/internal/gateway/codex_images.go b/internal/gateway/codex_images.goindex e7c2e844b..abf573319 100644--- a/internal/gateway/codex_images.go+++ b/internal/gateway/codex_images.go@@ -7,6 +7,7 @@ import ("encoding/base64""encoding/binary""image"+ "image/color""image/draw""image/gif""image/jpeg"@@ -134,6 +135,15 @@ type fitJob struct {// Retina one.const fitWorkers = 4+// fitBudget is how many bytes of decoded image one request may hold at+// once, across fitWorkers of them. fitWorkers alone bounds nothing: four+// images made side by side, each up to what one image may hold, is 4+// times codexImageMaxBytes, and a request that asks for that much is a+// process the system kills (yg6888 on Discord: a few agents at once put+// the heap past 300 MB with 1.5 MB bodies). A request's images are made in+// batches that fit here together, so the second batch waits for the first.+const fitBudget = 512 << 20+// fitAll fits each job's image: as it was kept (fitted), or else made// again, the images not kept made side by side.func fitAll(jobs []fitJob) {@@ -164,9 +174,53 @@ func fitAll(jobs []fitJob) {order = append(order, i)}}+ // what each will hold once made, from its header, so that the batches+ // below hold no more than fitBudget between them+ holds := make([]int, len(order))+ for n, i := range order {+ holds[n] = fitHolds(jobs[i].url)+ }+ for _, b := range fitInHand(holds, fitBudget) {+ fitBatch(jobs, order[b[0]:b[1]], made)+ }+ for _, i := range order {+ fitted.put(jobs[i].key, *made[jobs[i].key])+ }+ for _, i := range todo {+ e := made[jobs[i].key]+ jobs[i].with, jobs[i].changed = e.url(), e.data != nil+ }+}++// fitInHand splits what is to be made into the batches made one after+// another: no batch holds more than budget once its images are made,+// except one image that holds more than the budget by itself, which is+// made alone and turned away by codexImageRefit. Each batch is a pair of+// where it starts and where it ends in holds.+func fitInHand(holds []int, budget int) [][2]int {+ var out [][2]int+ for at := 0; at < len(holds); {+ room, end := budget, at+ for ; end < len(holds); end++ {+ if holds[end] > room {+ break+ }+ room -= holds[end]+ }+ if end == at {+ end+++ }+ out = append(out, [2]int{at, end})+ at = end+ }+ return out+}++// fitBatch makes the images of a batch, fitWorkers of them at once.+func fitBatch(jobs []fitJob, batch []int, made map[[32]byte]*fitEntry) {next := make(chan int)var wg sync.WaitGroup- for range min(fitWorkers, runtime.GOMAXPROCS(0), len(order)) {+ for range min(fitWorkers, runtime.GOMAXPROCS(0), len(batch)) {wg.Add(1)go func() {defer wg.Done()@@ -176,18 +230,11 @@ func fitAll(jobs []fitJob) {}}()}- for _, i := range order {+ for _, i := range batch {next <- i}close(next)wg.Wait()- for _, i := range order {- fitted.put(jobs[i].key, *made[jobs[i].key])- }- for _, i := range todo {- e := made[jobs[i].key]- jobs[i].with, jobs[i].changed = e.url(), e.data != nil- }}type imageLimits struct{ side, patches int }@@ -232,9 +279,79 @@ func (l imageLimits) size(w, h int) (int, int) {return max(int(math.Floor(fw*scale)), 1), max(int(math.Floor(fh*scale)), 1)}-// codexImageMaxPixels keeps a request from decoding an image too large-// to hold: one past it goes as it came.-const codexImageMaxPixels = 80 << 20+// codexImageMaxBytes is what one image may hold once decoded, its pixels+// and the bytes they came in: past it the image goes as it came.+//+// It used to be 80<<20 pixels, which is not what a decode holds. A header+// of a few bytes says what size an image is and the decoder allocates what+// it says before it reads a pixel, so 65 bytes declaring 8000x8000 at 16+// bits a channel held 512 MB, and four of them in one request held 1.8 GB+// (measured with runtime.MemStats). The bits a channel costs are the+// format's, and imageHolds reads them from the header.+const codexImageMaxBytes = 320 << 20++// holdsMost is the most imageHolds tells: one past the budget, so that a+// declared size no int holds still counts as too large.+const holdsMost = codexImageMaxBytes + 1++// imageHolds is what decoding an image of this header config holds: its+// pixels in the image its format makes of them. That's the format's own+// check above, so that the bytes a request holds follow the size it+// declares and the bits each channel costs, not the pixels alone.+func imageHolds(cfg image.Config, format string) (int, bool) {+ if cfg.Width <= 0 || cfg.Height <= 0 {+ return 0, false+ }+ // 4 bytes a pixel for RGBA, which anything with colour or alpha is+ // decoded into; 1 for grey; 2 for grey with 16 bits a channel and 8+ // for colour with 16, as image/png's readImagePass allocates them.+ bytesPerPixel := 4+ switch {+ case cfg.ColorModel == color.GrayModel:+ bytesPerPixel = 1+ case format == "png" && cfg.ColorModel == color.Gray16Model:+ bytesPerPixel = 2+ case format == "png" && (cfg.ColorModel == color.RGBA64Model || cfg.ColorModel == color.NRGBA64Model):+ bytesPerPixel = 8+ }+ pixels := int64(cfg.Width) * int64(cfg.Height)+ if pixels <= 0 || pixels*int64(bytesPerPixel) > holdsMost {+ return holdsMost, true+ }+ return int(pixels) * bytesPerPixel, true+}++// fitsHead is how much of an image's base64 fitHolds reads to tell its+// header: enough for a PNG's chunks and a JPEG's segments, and little+// enough that asking costs nothing.+const fitsHead = 4 << 10++// fitHolds is what making an image of url will hold, from its header+// alone, so that nothing is decoded to find out. An image it can't read+// holds nothing and isn't made.+func fitHolds(url string) int {+ comma := strings.IndexByte(url, ',')+ if comma < 0 || !strings.HasSuffix(url[:comma], ";base64") {+ return 0+ }+ b64 := url[comma+1:]+ if len(b64) > fitsHead {+ b64 = b64[:fitsHead&^3] // whole groups of four, as base64 reads them+ }+ head, err := base64.StdEncoding.DecodeString(b64)+ if err != nil {+ return 0+ }+ cfg, format, err := image.DecodeConfig(bytes.NewReader(head))+ if err != nil {+ return 0+ }+ holds, ok := imageHolds(cfg, format)+ if !ok {+ return 0+ }+ return holds+}// fitEntry is what an image is sent as: data in mime, or, with no data,// the image as it came.@@ -347,12 +464,21 @@ var refits atomic.Int64// codexImageRefit is an image's base64 resized and written again, or an// entry with no data when it goes as it came.func codexImageRefit(b64 string, l imageLimits) fitEntry {+ if len(b64) > codexImageMaxBytes {+ return fitEntry{} // more than it may hold before it is decoded+ }raw, err := base64.StdEncoding.DecodeString(b64)if err != nil {return fitEntry{}}cfg, format, err := image.DecodeConfig(bytes.NewReader(raw))- if err != nil || cfg.Width*cfg.Height > codexImageMaxPixels {+ if err != nil {+ return fitEntry{}+ }+ // what the pixels and the bytes they came in will hold together, so+ // that a header of a few bytes can't ask for a gigabyte+ holds, ok := imageHolds(cfg, format)+ if !ok || len(raw)+holds > codexImageMaxBytes {return fitEntry{}}w, h := l.size(cfg.Width, cfg.Height)diff --git a/internal/gateway/codex_images_test.go b/internal/gateway/codex_images_test.goindex 4c0e815cf..bdc5dc0a3 100644--- a/internal/gateway/codex_images_test.go+++ b/internal/gateway/codex_images_test.go@@ -4,8 +4,10 @@ import ("bytes""container/list""encoding/base64"+ "encoding/binary""encoding/json""fmt"+ "hash/crc32""image""image/color""image/jpeg"@@ -14,6 +16,7 @@ import ("math/rand/v2""net/http""net/http/httptest"+ "runtime""strings""testing""time"@@ -377,6 +380,215 @@ func fitKeyEntry(t *testing.T, shot string) fitEntry {return e}+// pngHeader is a PNG whose header declares w by h at depth bits a channel+// and colour, with a valid zlib stream of no pixels after it: what a+// decode reads the size from, before it reads a pixel. Go's png decodes it+// as far as allocating the image the header declares, which is where a+// header alone costs its bytes.+func pngHeader(w, h uint32, depth, colour byte) []byte {+ var b bytes.Buffer+ b.Write([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'})+ be := func(v any) { _ = binary.Write(&b, binary.BigEndian, v) }+ var ihdr bytes.Buffer+ _ = binary.Write(&ihdr, binary.BigEndian, w)+ _ = binary.Write(&ihdr, binary.BigEndian, h)+ ihdr.Write([]byte{depth, colour, 0, 0, 0})+ for _, c := range []struct {+ kind string+ body []byte+ }{+ {"IHDR", ihdr.Bytes()},+ // a valid zlib stream of no pixel data, so that a decode reads+ // the header and allocates what it says+ {"IDAT", []byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01}},+ {"IEND", nil},+ } {+ be(uint32(len(c.body)))+ head := append([]byte(c.kind), c.body...)+ b.Write(head)+ be(crc32.ChecksumIEEE(head))+ }+ return b.Bytes()+}++func bombURL(w, h uint32, depth, colour byte) string {+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(pngHeader(w, h, depth, colour))+}++// allocDuring is the bytes f allocates, from before it runs to after,+// which is what a decode of a size it was given costs whether or not it+// then finds the pixels.+func allocDuring(f func()) uint64 {+ runtime.GC()+ var before, after runtime.MemStats+ runtime.ReadMemStats(&before)+ f()+ runtime.ReadMemStats(&after)+ return after.TotalAlloc - before.TotalAlloc+}++// A header of a few dozen bytes is all it takes to ask a decode for+// hundreds of megabytes: it declares the size, and png allocates that+// before it reads a pixel, so the decode is paid for even though it then+// fails. 110 bytes declaring 8000x8000 at 16 bits a channel held 488 MB,+// and four such images in one request held 1.7 GB: a process the system+// kills, off 260 bytes of an upload. What a decode may hold is judged+// from the declaration now, before it decodes, and it is the same measure+// for every format (codexImageMaxBytes, four bytes a pixel for anything+// with colour, as image/png allocates it).+func TestCodexAHeaderAloneCantHoldPastTheBudget(t *testing.T) {+ if testing.Short() {+ t.Skip("measures the bytes a decode holds")+ }+ for _, c := range []struct {+ w, h uint32+ depth byte+ colour byte+ what string+ }{+ {8000, 8000, 8, 6, "8000x8000 at 8 bits a channel: 256 MB of pixels"},+ {8000, 8000, 16, 6, "8000x8000 at 16 bits a channel: 512 MB of pixels"},+ {70000, 70000, 8, 6, "70000x70000: more than an int holds"},+ } {+ url := bombURL(c.w, c.h, c.depth, c.colour)+ if n := len(url); n > 1024 {+ t.Fatalf("%s: the upload is %d bytes, not a header alone", c.what, n)+ }+ b64 := url[strings.IndexByte(url, ',')+1:]+ held := allocDuring(func() {+ codexImageRefit(b64, imageLimits{2048, 2500})+ })+ if held > codexImageMaxBytes {+ t.Errorf("%s: %d bytes of upload held %d MB, past the %d MB a decode may hold",+ c.what, len(url), held>>20, codexImageMaxBytes>>20)+ }+ }+}++// The same, in one request: four headers each declaring a size it may not+// hold, which codexImagesFit used to make side by side.+func TestCodexFourHeadersInOneRequestCantHoldTheHeap(t *testing.T) {+ const allowed = 128 << 20+ if testing.Short() {+ t.Skip("measures the bytes a decode holds")+ }+ // four *distinct* images: one entry is made per image, so the same data+ // URL four times is made once+ urls := make([]string, 4)+ for i := range urls {+ urls[i] = bombURL(7000+uint32(i), 8000, 16, 6)+ }+ body := computerUseRequest(urls)+ held := allocDuring(func() {+ codexImagesFit(codexAccount(), provider.Responses, body)+ })+ if held > allowed {+ t.Errorf("%d bytes of upload, sent as 4 images in one request, held %d MB, past %d MB",+ len(body), held>>20, allowed>>20)+ }+}++// The budget the bytes above are counted against is the one a real+// screenshot of the size Codex sends still fits under: what it holds once+// decoded, so that a genuine image and a header alone are judged by the+// same measure.+func TestCodexRealScreenshotsFitBytesTheyHold(t *testing.T) {+ for _, c := range []struct{ w, h int }{+ {3456, 2234}, // ylorn's, the size Codex's own client sends+ {4032, 3024},+ {6000, 4000},+ } {+ for _, format := range []string{"png", "jpeg"} {+ url := shotURL(t, shotImage(c.w, c.h), format)+ raw := rawOf(t, url)+ cfg, f, err := image.DecodeConfig(bytes.NewReader(raw))+ if err != nil {+ t.Fatal(err)+ }+ holds, ok := imageHolds(cfg, f)+ if !ok || len(raw)+holds > codexImageMaxBytes {+ t.Fatalf("%dx%d %s holds %d MB: a real screenshot is turned away", c.w, c.h, format, holds>>20)+ }+ e := codexImageRefit(url[strings.IndexByte(url, ',')+1:], imageLimits{2048, 2500})+ if e.data == nil {+ t.Fatalf("%dx%d %s went as it came", c.w, c.h, format)+ }+ if m, _ := shotDecode(t, "data:"+e.mime+";base64,"+base64.StdEncoding.EncodeToString(e.data)); m == nil {+ t.Fatalf("%dx%d %s: what was made isn't an image", c.w, c.h, format)+ }+ }+ }+}++func rawOf(t *testing.T, url string) []byte {+ t.Helper()+ raw, err := base64.StdEncoding.DecodeString(url[strings.IndexByte(url, ',')+1:])+ if err != nil {+ t.Fatal(err)+ }+ return raw+}++// What one request may hold is batched: no more than fitBudget is made at+// once, however many images the request carries, so that a request of 100+// screenshots holds what four do and not what a hundred do.+func TestCodexFittingHoldsTheBudget(t *testing.T) {+ const mb = 1 << 20+ for _, c := range []struct {+ what string+ holds []int+ budget int+ want [][2]int+ }{+ {"four small ones, all at once", []int{10 * mb, 10 * mb, 10 * mb, 10 * mb}, 100 * mb, [][2]int{{0, 4}}},+ {"one more than the budget, alone", []int{10 * mb, 250 * mb, 10 * mb}, 100 * mb, [][2]int{{0, 1}, {1, 2}, {2, 3}}},+ {"the second waits for the first", []int{60 * mb, 60 * mb}, 100 * mb, [][2]int{{0, 1}, {1, 2}}},+ {"a batch breaks where the next wouldn't fit", []int{40 * mb, 40 * mb, 40 * mb, 40 * mb, 10 * mb}, 100 * mb, [][2]int{{0, 2}, {2, 5}}},+ {"a batch breaks where the next wouldn't fit, exactly", []int{40 * mb, 40 * mb, 40 * mb, 30 * mb}, 100 * mb, [][2]int{{0, 2}, {2, 4}}},+ {"one past the budget on its own, alone", []int{400 * mb, 10 * mb}, 100 * mb, [][2]int{{0, 1}, {1, 2}}},+ {"nothing to make", nil, 100 * mb, nil},+ } {+ got := fitInHand(c.holds, c.budget)+ if len(got) != len(c.want) {+ t.Errorf("%s: %d batches %v, want %d %v", c.what, len(got), got, len(c.want), c.want)+ continue+ }+ for i := range got {+ if got[i] != c.want[i] {+ t.Errorf("%s: batch %d is %v, want %v", c.what, i, got[i], c.want[i])+ }+ }+ // and what any batch holds is the budget, or one image alone+ at := 0+ for _, b := range got {+ if b[0] != at {+ t.Errorf("%s: batch %v leaves a gap after %d", c.what, b, at)+ }+ at = b[1]+ sum := 0+ for _, h := range c.holds[b[0]:b[1]] {+ sum += h+ }+ if sum > c.budget && b[1]-b[0] != 1 {+ t.Errorf("%s: batch %v holds %d MB past the budget of %d MB", c.what, b, sum/mb, c.budget/mb)+ }+ }+ }+}++// A request whose images fit the budget is made as it was: the batching+// changes nothing for the screenshots a real turn carries.+func TestCodexFittingLeavesATurnThatFitsAsItWas(t *testing.T) {+ made := freshFitted(t, fittedBudget)+ shots := wideShots(t, 8)+ if out := codexImagesFit(codexAccount(), provider.Responses, computerUseRequest(shots)); len(out) == 0 {+ t.Fatal("no reply body")+ }+ if n := made(); n != 8 {+ t.Errorf("%d of 8 images made", n)+ }+}+// What no request has sent for fittedIdle is let go.func TestCodexFittedImagesLetGoWhenIdle(t *testing.T) {made := freshFitted(t, fittedBudget)