Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a conversation's list is looked at where it sits in the body, not copied (#4004)

yetone56d26f0bba65
diff --git a/internal/gateway/anthropic_shapes.go b/internal/gateway/anthropic_shapes.go
index 7e3b20e57..8cc6749e6 100644
--- a/internal/gateway/anthropic_shapes.go
+++ b/internal/gateway/anthropic_shapes.go
@@ -191,7 +191,7 @@ func hasTurnEffort(body []byte) bool {
return false
}
found := false
- gjson.GetBytes(body, "messages").ForEach(func(_, m gjson.Result) bool {
+ getIn(body, "messages").ForEach(func(_, m gjson.Result) bool {
found = m.Get("output_config").Exists()
return !found
})
@@ -325,7 +325,7 @@ func hasDeferredTools(body []byte) bool {
return false
}
found := false
- gjson.GetBytes(body, "messages").ForEach(func(_, m gjson.Result) bool {
+ getIn(body, "messages").ForEach(func(_, m gjson.Result) bool {
m.Get("content").ForEach(func(_, b gjson.Result) bool {
found = deferredBlock(b.Get("type").String())
return !found
@@ -449,7 +449,7 @@ func hasRefsOnlyResult(body []byte) bool {
return false
}
found := false
- gjson.GetBytes(body, "messages").ForEach(func(_, m gjson.Result) bool {
+ getIn(body, "messages").ForEach(func(_, m gjson.Result) bool {
m.Get("content").ForEach(func(_, b gjson.Result) bool {
found = b.Get("type").String() == "tool_result" && refsOnly(b.Get("content"))
return !found
diff --git a/internal/gateway/cline_pin.go b/internal/gateway/cline_pin.go
index 6a528b3b1..a05969dd3 100644
--- a/internal/gateway/cline_pin.go
+++ b/internal/gateway/cline_pin.go
@@ -57,7 +57,7 @@ func clinePin(p provider.Provider, to provider.Protocol, body []byte) []byte {
// chatImages is whether a Chat body carries an image part
func chatImages(body []byte) bool {
found := false
- gjson.GetBytes(body, "messages").ForEach(func(_, m gjson.Result) bool {
+ getIn(body, "messages").ForEach(func(_, m gjson.Result) bool {
m.Get("content").ForEach(func(_, part gjson.Result) bool {
if t := part.Get("type").String(); t == "image_url" || t == "image" || t == "input_image" {
found = true
diff --git a/internal/gateway/codex_images.go b/internal/gateway/codex_images.go
index fd022a975..8d7d638ac 100644
--- a/internal/gateway/codex_images.go
+++ b/internal/gateway/codex_images.go
@@ -50,7 +50,7 @@ func codexImagesFit(p provider.Provider, to provider.Protocol, body []byte) []by
// every image: in a Computer Use turn of 160 screenshots, gigabytes of
// JSON read for one request (fflake33, Discord).
var jobs []fitJob
- gjson.GetBytes(body, "input").ForEach(func(_, item gjson.Result) bool {
+ getIn(body, "input").ForEach(func(_, item gjson.Result) bool {
if !strings.Contains(item.Raw, "input_image") {
return true
}
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.go
index 01ca843cb..28e29b504 100644
--- a/internal/gateway/fastjson.go
+++ b/internal/gateway/fastjson.go
@@ -16,6 +16,8 @@ import (
"unicode/utf16"
"unicode/utf8"
"unsafe"
+
+ "github.com/tidwall/gjson"
)
// decodeJSON is json.Unmarshal(data, dst), made for a request body: one
@@ -1427,6 +1429,16 @@ func appendHTMLEscaped(dst, s []byte) []byte {
return append(dst, s[start:]...)
}
+// getIn is gjson.GetBytes read in place: what it finds points into body
+// rather than into a copy. GetBytes copies the value it finds, and for a
+// conversation's list that is the conversation, 45 MiB of a Codex turn
+// for one look at its items (#4004). A string kept from the result keeps
+// all of body: clone one that outlives the request, and never change body
+// while one is held.
+func getIn(body []byte, path string) gjson.Result {
+ return gjson.Get(unsafe.String(unsafe.SliceData(body), len(body)), path)
+}
+
// marshalBody is json.Marshal of a request body being built, byte for
// byte, its lists (the messages, the input items) written an item at a
// time into a buffer of the size they come to. json.Marshal writes the
diff --git a/internal/gateway/native_compaction.go b/internal/gateway/native_compaction.go
index 5f3bb968f..f1f671f22 100644
--- a/internal/gateway/native_compaction.go
+++ b/internal/gateway/native_compaction.go
@@ -48,7 +48,7 @@ func compactionTriggerIn(body []byte) bool {
if !bytes.Contains(body, []byte(`"compaction_trigger"`)) {
return false
}
- for _, it := range gjson.GetBytes(body, "input").Array() {
+ for _, it := range getIn(body, "input").Array() {
if it.Get("type").Str == "compaction_trigger" {
return true
}
@@ -64,7 +64,7 @@ func sealedCompactions(body []byte) []string {
return nil
}
var out []string
- for _, it := range gjson.GetBytes(body, "input").Array() {
+ for _, it := range getIn(body, "input").Array() {
if !slices.Contains(compactionKinds, it.Get("type").Str) {
continue
}
diff --git a/internal/gateway/title_link.go b/internal/gateway/title_link.go
index 0c4bdc800..62f588d28 100644
--- a/internal/gateway/title_link.go
+++ b/internal/gateway/title_link.go
@@ -112,7 +112,7 @@ func titleUserText(content gjson.Result, title bool) string {
func titlePromptDigest(body []byte, title bool) string {
var digest string
i := 0
- input := gjson.GetBytes(body, "input")
+ input := getIn(body, "input")
if !input.IsArray() {
return ""
}
diff --git a/internal/gateway/tool_loop.go b/internal/gateway/tool_loop.go
index dd9951e63..e29e0b47a 100644
--- a/internal/gateway/tool_loop.go
+++ b/internal/gateway/tool_loop.go
@@ -98,7 +98,7 @@ const tailItems = 8 * toolLoopRepeats
// took ~30ms and 20MB a turn of 1.5MB; gjson finds the items without
// decoding them. A body whose conversation isn't such a list is as it came.
func tailOf(from provider.Protocol, body []byte) []byte {
- list := gjson.GetBytes(body, conversationKey[from])
+ list := getIn(body, conversationKey[from])
if !list.IsArray() || list.Index <= 0 {
return body
}
diff --git a/internal/gateway/tool_loop_test.go b/internal/gateway/tool_loop_test.go
index 34654e8a2..b6f47a991 100644
--- a/internal/gateway/tool_loop_test.go
+++ b/internal/gateway/tool_loop_test.go
@@ -329,3 +329,24 @@ func BenchmarkToolLoopCheck(b *testing.B) {
}
}
}
+
+// The conversation's tail is read where it sits in the body: GetBytes
+// copied the whole input list to find the last items, 45 MiB of a Codex
+// turn on every request (#4004).
+func TestTailOfReadsInPlace(t *testing.T) {
+ item := `{"type":"function_call_output","call_id":"c%d","output":"` + strings.Repeat("o", 20<<10) + `"}`
+ items := make([]string, 200)
+ for i := range items {
+ items[i] = fmt.Sprintf(item, i)
+ }
+ body := []byte(`{"model":"m","input":[` + strings.Join(items, ",") + `],"stream":true}`)
+ var tail []byte
+ n := allocated(func() { tail = tailOf(provider.Responses, body) })
+ want := `{"model":"m","input":[` + strings.Join(items[len(items)-tailItems:], ",") + `],"stream":true}`
+ if string(tail) != want {
+ t.Fatalf("tail is %.200s…", tail)
+ }
+ if n > uint64(len(want))+uint64(len(body))/4 {
+ t.Fatalf("%d bytes allocated for a %d byte tail of a %d byte body", n, len(want), len(body))
+ }
+}