Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a request's turns are looked through for images in place, before any copy (#3725)

yetonefcbf618ebb7b
diff --git a/internal/gateway/image_input.go b/internal/gateway/image_input.go
index 5cf69f9c2..71c9c02c9 100644
--- a/internal/gateway/image_input.go
+++ b/internal/gateway/image_input.go
@@ -67,10 +67,6 @@ func (im imageAt) Current() bool { return im.Last && !im.Tool }
// block it returns in the image's place (nil keeps it). An error from swap
// ends the walk. The body is returned as it was when nothing was swapped.
func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.RawMessage, error)) ([]byte, error) {
- var request map[string]json.RawMessage
- if decodeJSON(body, &request) != nil {
- return body, nil
- }
turnsKey, contentKey := "messages", "content"
switch proto {
case provider.Responses:
@@ -78,10 +74,15 @@ func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.R
case provider.Gemini:
turnsKey, contentKey = "contents", "parts"
}
- if raw := request[turnsKey]; len(raw) > 0 && raw[0] == '[' && !anyImageBlock(proto, raw, contentKey) {
+ if !anyImageTurn(proto, body, turnsKey, contentKey) {
// Most requests have no image, and this walk runs several times
// on each: decoding every turn and block cost ~2.6 MB for each
- // 1.5 MB conversation (yg6888 on X).
+ // 1.5 MB conversation (yg6888 on X), and the request's top level
+ // alone was a copy of it each time (#3725).
+ return body, nil
+ }
+ var request map[string]json.RawMessage
+ if decodeJSON(body, &request) != nil {
return body, nil
}
var turns []json.RawMessage
@@ -125,12 +126,34 @@ func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.R
return json.Marshal(request)
}
+// anyImageTurn is whether body may hold a block walkImages gives to swap:
+// it is a JSON object whose turnsKey is a list anyImageBlock says yes to,
+// read in place. A key given twice is looked at each time.
+func anyImageTurn(proto provider.Protocol, body []byte, turnsKey, contentKey string) bool {
+ if !jsonValid(body) {
+ return false
+ }
+ raw := trimJSON(body)
+ if raw[0] != '{' {
+ return false
+ }
+ found := false
+ eachMember(raw, func(key, val []byte) bool {
+ if val[0] == '[' && jsonStringIs(key, turnsKey) {
+ found = anyImageBlock(proto, val, contentKey)
+ }
+ return !found
+ })
+ return found
+}
+
// anyImageBlock is whether turns, a valid JSON array, may hold a block
// walkImages gives to swap, read in place without decoding: a turn's
// contentKey (or a Responses function_call_output's output) that is a list
-// with a block among it of an image's type, an Anthropic tool_result, or
-// a Gemini part with inlineData, fileData or a functionResponse. It errs
-// toward yes, as for a key spelled twice.
+// with a block among it of an image's type, or one inside an Anthropic
+// tool_result's content or a Gemini functionResponse's parts, or a Gemini
+// part with inlineData or fileData. It errs toward yes, as for a key
+// spelled twice.
func anyImageBlock(proto provider.Protocol, turns []byte, contentKey string) bool {
found := false
eachElement(turns, func(turn []byte) bool {
@@ -163,15 +186,35 @@ func anyImageIn(proto provider.Protocol, blocks []byte) bool {
if block[0] != '{' {
return true
}
+ // a tool's images are in its own blocks, looked into as
+ // swapImageBlocks does: most agents' turns are tool results,
+ // nearly all text (#3725)
+ var inner []byte
+ toolResult := false
eachMember(block, func(key, val []byte) bool {
switch {
+ case proto == provider.Gemini && jsonStringIs(key, "functionResponse"):
+ if val[0] == '{' {
+ eachMember(val, func(k, v []byte) bool {
+ if jsonStringIs(k, "parts") {
+ inner = v // the last, as a decode keeps
+ }
+ return true
+ })
+ }
case proto == provider.Gemini:
- found = jsonStringIs(key, "inlineData") || jsonStringIs(key, "fileData") || jsonStringIs(key, "functionResponse")
+ found = jsonStringIs(key, "inlineData") || jsonStringIs(key, "fileData")
case jsonStringIs(key, "type"):
- found = jsonStringIs(val, kind) || (proto == provider.Anthropic && jsonStringIs(val, "tool_result"))
+ found = jsonStringIs(val, kind)
+ toolResult = toolResult || (proto == provider.Anthropic && jsonStringIs(val, "tool_result"))
+ case proto == provider.Anthropic && jsonStringIs(key, "content"):
+ inner = val
}
return !found
})
+ if !found && len(inner) > 0 && inner[0] == '[' && (proto == provider.Gemini || toolResult) {
+ found = anyImageIn(proto, inner)
+ }
return !found
})
return found
diff --git a/internal/gateway/image_input_test.go b/internal/gateway/image_input_test.go
index 6f58d124f..68fc49090 100644
--- a/internal/gateway/image_input_test.go
+++ b/internal/gateway/image_input_test.go
@@ -3,6 +3,7 @@ package gateway
import (
"context"
"encoding/json"
+ "fmt"
"io"
"net/http"
"net/http/httptest"
@@ -585,3 +586,52 @@ func TestImageWalkReadsTextOnlyTurnsInPlace(t *testing.T) {
}
}
}
+
+// An agent's turns are mostly tool results, and nearly all of them text:
+// the image walk looks into each one where it is, as it does a turn,
+// rather than decoding the request whenever a tool_result or a
+// functionResponse is there (#3725). An image a tool gave back is still
+// found, however its block is spelled.
+func TestImageWalkReadsToolResultsInPlace(t *testing.T) {
+ var turns []string
+ for i := range 60 {
+ turns = append(turns,
+ `{"role":"assistant","content":[{"type":"tool_use","id":"t`+fmt.Sprint(i)+`","name":"Read","input":{"path":"a.go"}}]}`,
+ `{"role":"user","content":[{"type":"tool_result","tool_use_id":"t`+fmt.Sprint(i)+`","content":[{"type":"text","text":"`+strings.Repeat("package a ", 100)+`"}]},{"type":"tool_result","tool_use_id":"u","content":"done"}]}`)
+ }
+ body := []byte(`{"model":"m","messages":[` + strings.Join(turns, ",") + `]}`)
+ if hasImage(provider.Anthropic, body) {
+ t.Fatal("an image found in a text-only body")
+ }
+ if per := testing.AllocsPerRun(5, func() { hasImage(provider.Anthropic, body) }); per > 4 {
+ t.Fatalf("looking for an image in a %d-byte body of text tool results took %v allocations", len(body), per)
+ }
+ gem := []byte(`{"contents":[{"role":"user","parts":[{"functionResponse":{"name":"f","response":{"out":"` + strings.Repeat("x", 4000) + `"}}}]}]}`)
+ if hasImage(provider.Gemini, gem) {
+ t.Fatal("an image found in a Gemini text-only function response")
+ }
+ if per := testing.AllocsPerRun(5, func() { hasImage(provider.Gemini, gem) }); per > 4 {
+ t.Fatalf("looking for an image in a Gemini function response took %v allocations", per)
+ }
+
+ img := `{"type":"image","source":{"type":"url","url":"https://x/a.png"}}`
+ for i, c := range []struct {
+ proto provider.Protocol
+ body string
+ want bool
+ }{
+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"content":[` + img + `],"type":"tool_result","tool_use_id":"t"}]}]}`, true},
+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"tool_result","content":[{"type":"text","text":"a"}],"content":[` + img + `]}]}]}`, true},
+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"tool_result","content":[{"type":"tool_result","content":[` + img + `]}]}]}]}`, true},
+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"%u0074ype":"tool_result","content":[` + img + `]}]}]}`, true},
+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"text","content":[` + img + `]}]}]}`, false},
+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"tool_result","content":"see a.png"}]}]}`, false},
+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"functionResponse":{"parts":[{"inlineData":{"mimeType":"image/png","data":"AAAA"}}],"name":"f"}}]}]}`, true},
+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"functionResponse":{"name":"f","parts":[{"functionResponse":{"parts":[{"fileData":{"fileUri":"gs://a"}}]}}]}}]}]}`, true},
+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"functionResponse":{"name":"f","parts":[{"text":"a"}]}}]}]}`, false},
+ } {
+ if got := hasImage(c.proto, []byte(strings.ReplaceAll(c.body, "%u", "\x5cu"))); got != c.want {
+ t.Errorf("case %d (%v): hasImage = %v, want %v", i, c.proto, got, c.want)
+ }
+ }
+}