Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a request's turns are looked through for images in place, before any copy (#3725)
yetonefcbf618ebb7b
diff --git a/internal/gateway/image_input.go b/internal/gateway/image_input.goindex 5cf69f9c2..71c9c02c9 100644--- a/internal/gateway/image_input.go+++ b/internal/gateway/image_input.go@@ -67,10 +67,6 @@ func (im imageAt) Current() bool { return im.Last && !im.Tool }// block it returns in the image's place (nil keeps it). An error from swap// ends the walk. The body is returned as it was when nothing was swapped.func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.RawMessage, error)) ([]byte, error) {- var request map[string]json.RawMessage- if decodeJSON(body, &request) != nil {- return body, nil- }turnsKey, contentKey := "messages", "content"switch proto {case provider.Responses:@@ -78,10 +74,15 @@ func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.Rcase provider.Gemini:turnsKey, contentKey = "contents", "parts"}- if raw := request[turnsKey]; len(raw) > 0 && raw[0] == '[' && !anyImageBlock(proto, raw, contentKey) {+ if !anyImageTurn(proto, body, turnsKey, contentKey) {// Most requests have no image, and this walk runs several times// on each: decoding every turn and block cost ~2.6 MB for each- // 1.5 MB conversation (yg6888 on X).+ // 1.5 MB conversation (yg6888 on X), and the request's top level+ // alone was a copy of it each time (#3725).+ return body, nil+ }+ var request map[string]json.RawMessage+ if decodeJSON(body, &request) != nil {return body, nil}var turns []json.RawMessage@@ -125,12 +126,34 @@ func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.Rreturn json.Marshal(request)}+// anyImageTurn is whether body may hold a block walkImages gives to swap:+// it is a JSON object whose turnsKey is a list anyImageBlock says yes to,+// read in place. A key given twice is looked at each time.+func anyImageTurn(proto provider.Protocol, body []byte, turnsKey, contentKey string) bool {+ if !jsonValid(body) {+ return false+ }+ raw := trimJSON(body)+ if raw[0] != '{' {+ return false+ }+ found := false+ eachMember(raw, func(key, val []byte) bool {+ if val[0] == '[' && jsonStringIs(key, turnsKey) {+ found = anyImageBlock(proto, val, contentKey)+ }+ return !found+ })+ return found+}+// anyImageBlock is whether turns, a valid JSON array, may hold a block// walkImages gives to swap, read in place without decoding: a turn's// contentKey (or a Responses function_call_output's output) that is a list-// with a block among it of an image's type, an Anthropic tool_result, or-// a Gemini part with inlineData, fileData or a functionResponse. It errs-// toward yes, as for a key spelled twice.+// with a block among it of an image's type, or one inside an Anthropic+// tool_result's content or a Gemini functionResponse's parts, or a Gemini+// part with inlineData or fileData. It errs toward yes, as for a key+// spelled twice.func anyImageBlock(proto provider.Protocol, turns []byte, contentKey string) bool {found := falseeachElement(turns, func(turn []byte) bool {@@ -163,15 +186,35 @@ func anyImageIn(proto provider.Protocol, blocks []byte) bool {if block[0] != '{' {return true}+ // a tool's images are in its own blocks, looked into as+ // swapImageBlocks does: most agents' turns are tool results,+ // nearly all text (#3725)+ var inner []byte+ toolResult := falseeachMember(block, func(key, val []byte) bool {switch {+ case proto == provider.Gemini && jsonStringIs(key, "functionResponse"):+ if val[0] == '{' {+ eachMember(val, func(k, v []byte) bool {+ if jsonStringIs(k, "parts") {+ inner = v // the last, as a decode keeps+ }+ return true+ })+ }case proto == provider.Gemini:- found = jsonStringIs(key, "inlineData") || jsonStringIs(key, "fileData") || jsonStringIs(key, "functionResponse")+ found = jsonStringIs(key, "inlineData") || jsonStringIs(key, "fileData")case jsonStringIs(key, "type"):- found = jsonStringIs(val, kind) || (proto == provider.Anthropic && jsonStringIs(val, "tool_result"))+ found = jsonStringIs(val, kind)+ toolResult = toolResult || (proto == provider.Anthropic && jsonStringIs(val, "tool_result"))+ case proto == provider.Anthropic && jsonStringIs(key, "content"):+ inner = val}return !found})+ if !found && len(inner) > 0 && inner[0] == '[' && (proto == provider.Gemini || toolResult) {+ found = anyImageIn(proto, inner)+ }return !found})return founddiff --git a/internal/gateway/image_input_test.go b/internal/gateway/image_input_test.goindex 6f58d124f..68fc49090 100644--- a/internal/gateway/image_input_test.go+++ b/internal/gateway/image_input_test.go@@ -3,6 +3,7 @@ package gatewayimport ("context""encoding/json"+ "fmt""io""net/http""net/http/httptest"@@ -585,3 +586,52 @@ func TestImageWalkReadsTextOnlyTurnsInPlace(t *testing.T) {}}}++// An agent's turns are mostly tool results, and nearly all of them text:+// the image walk looks into each one where it is, as it does a turn,+// rather than decoding the request whenever a tool_result or a+// functionResponse is there (#3725). An image a tool gave back is still+// found, however its block is spelled.+func TestImageWalkReadsToolResultsInPlace(t *testing.T) {+ var turns []string+ for i := range 60 {+ turns = append(turns,+ `{"role":"assistant","content":[{"type":"tool_use","id":"t`+fmt.Sprint(i)+`","name":"Read","input":{"path":"a.go"}}]}`,+ `{"role":"user","content":[{"type":"tool_result","tool_use_id":"t`+fmt.Sprint(i)+`","content":[{"type":"text","text":"`+strings.Repeat("package a ", 100)+`"}]},{"type":"tool_result","tool_use_id":"u","content":"done"}]}`)+ }+ body := []byte(`{"model":"m","messages":[` + strings.Join(turns, ",") + `]}`)+ if hasImage(provider.Anthropic, body) {+ t.Fatal("an image found in a text-only body")+ }+ if per := testing.AllocsPerRun(5, func() { hasImage(provider.Anthropic, body) }); per > 4 {+ t.Fatalf("looking for an image in a %d-byte body of text tool results took %v allocations", len(body), per)+ }+ gem := []byte(`{"contents":[{"role":"user","parts":[{"functionResponse":{"name":"f","response":{"out":"` + strings.Repeat("x", 4000) + `"}}}]}]}`)+ if hasImage(provider.Gemini, gem) {+ t.Fatal("an image found in a Gemini text-only function response")+ }+ if per := testing.AllocsPerRun(5, func() { hasImage(provider.Gemini, gem) }); per > 4 {+ t.Fatalf("looking for an image in a Gemini function response took %v allocations", per)+ }++ img := `{"type":"image","source":{"type":"url","url":"https://x/a.png"}}`+ for i, c := range []struct {+ proto provider.Protocol+ body string+ want bool+ }{+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"content":[` + img + `],"type":"tool_result","tool_use_id":"t"}]}]}`, true},+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"tool_result","content":[{"type":"text","text":"a"}],"content":[` + img + `]}]}]}`, true},+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"tool_result","content":[{"type":"tool_result","content":[` + img + `]}]}]}]}`, true},+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"%u0074ype":"tool_result","content":[` + img + `]}]}]}`, true},+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"text","content":[` + img + `]}]}]}`, false},+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"tool_result","content":"see a.png"}]}]}`, false},+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"functionResponse":{"parts":[{"inlineData":{"mimeType":"image/png","data":"AAAA"}}],"name":"f"}}]}]}`, true},+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"functionResponse":{"name":"f","parts":[{"functionResponse":{"parts":[{"fileData":{"fileUri":"gs://a"}}]}}]}}]}]}`, true},+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"functionResponse":{"name":"f","parts":[{"text":"a"}]}}]}]}`, false},+ } {+ if got := hasImage(c.proto, []byte(strings.ReplaceAll(c.body, "%u", "\x5cu"))); got != c.want {+ t.Errorf("case %d (%v): hasImage = %v, want %v", i, c.proto, got, c.want)+ }+ }+}