Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a translated body lets its built conversation go as it fills (#4004)

yetone8eb7f5565bda
diff --git a/internal/gateway/anthropic.go b/internal/gateway/anthropic.go
index 4497f4e6d..a445f15ff 100644
--- a/internal/gateway/anthropic.go
+++ b/internal/gateway/anthropic.go
@@ -1099,7 +1099,7 @@ func anthropicBody(r *Request, model, host string, replay bool) []byte {
out["tool_choice"] = tc
}
}
- b, _ := marshalBody(out)
+ b, _ := marshalBody(out, "messages")
return b
}
diff --git a/internal/gateway/chat.go b/internal/gateway/chat.go
index 7edff257c..0fca9b3ef 100644
--- a/internal/gateway/chat.go
+++ b/internal/gateway/chat.go
@@ -585,7 +585,7 @@ func buildChat(r *Request, model, host string, rejectTemp bool) []byte {
// OpenRouter's own search, for any of its models
out["plugins"] = []map[string]any{{"id": "web"}}
}
- b, _ := marshalBody(out)
+ b, _ := marshalBody(out, "messages")
return b
}
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.go
index 28e29b504..21f19715e 100644
--- a/internal/gateway/fastjson.go
+++ b/internal/gateway/fastjson.go
@@ -1447,16 +1447,27 @@ func getIn(body []byte, path string) gjson.Result {
// 316 MiB, the buffers it outgrew being garbage until the next GC (#4004,
// jihuayu's profile on a 2 GiB server). Here no buffer outgrows the
// largest item. A value json.Marshal refuses gets json.Marshal's answer.
-func marshalBody(out map[string]any) ([]byte, error) {
+//
+// spent names the list the builder made for this body alone (its
+// messages, its input items): each of its items is let go once written,
+// so the built conversation, strings joined from the request's parts
+// among it, is garbage while the body fills instead of held beside it to
+// the end (#4004). A value json.Marshal refuses is refused still, and the
+// list is then left part spent.
+func marshalBody(out map[string]any, spent string) ([]byte, error) {
keys := make([]string, 0, len(out))
for k := range out {
+ // a key json.Marshal writes another way, before anything is spent
+ if _, ok := appendJSONKey(nil, k, true); !ok {
+ return json.Marshal(out)
+ }
keys = append(keys, k)
}
slices.Sort(keys)
vals := make([][]json.RawMessage, len(keys))
n := 2
for i, k := range keys {
- items, ok := marshalItems(out[k])
+ items, ok := marshalItems(out[k], k == spent)
if !ok {
return json.Marshal(out)
}
@@ -1507,8 +1518,8 @@ func isSplitList(v any) bool {
// marshalItems is v as json.Marshal writes it: its items each on their
// own for a list isSplitList takes, or one piece; false when json.Marshal
-// refuses any of it.
-func marshalItems(v any) ([]json.RawMessage, bool) {
+// refuses any of it. spend sets each item written to its zero value.
+func marshalItems(v any, spend bool) ([]json.RawMessage, bool) {
if !isSplitList(v) {
b, err := json.Marshal(v)
return []json.RawMessage{b}, err == nil
@@ -1523,6 +1534,9 @@ func marshalItems(v any) ([]json.RawMessage, bool) {
return nil, false
}
items[i] = b
+ if spend {
+ rv.Index(i).SetZero()
+ }
}
return items, true
}
diff --git a/internal/gateway/gemini.go b/internal/gateway/gemini.go
index 2c031803a..03c3ef542 100644
--- a/internal/gateway/gemini.go
+++ b/internal/gateway/gemini.go
@@ -116,7 +116,7 @@ func buildGemini(r *Request, model string) ([]byte, error) {
return nil, err
}
req["model"] = model
- return marshalBody(req)
+ return marshalBody(req, "")
}
// geminiRequest is the request inside the Code Assist envelope that
diff --git a/internal/gateway/marshal_body_test.go b/internal/gateway/marshal_body_test.go
index 5b3b11d2b..7428028db 100644
--- a/internal/gateway/marshal_body_test.go
+++ b/internal/gateway/marshal_body_test.go
@@ -33,7 +33,7 @@ func TestMarshalBodyIsJSONMarshal(t *testing.T) {
}
for i, b := range bodies {
want, werr := json.Marshal(b)
- got, gerr := marshalBody(b)
+ got, gerr := marshalBody(b, "messages")
if (werr == nil) != (gerr == nil) || !bytes.Equal(got, want) {
t.Errorf("body %d:\n got %s (%v)\nwant %s (%v)", i, got, gerr, want, werr)
}
@@ -43,7 +43,7 @@ func TestMarshalBodyIsJSONMarshal(t *testing.T) {
{"x": math.Inf(1)},
{"ch": []any{"a", make(chan int)}},
} {
- if _, err := marshalBody(bad); err == nil {
+ if _, err := marshalBody(bad, ""); err == nil {
t.Errorf("marshalBody(%v) took what json.Marshal refuses", bad)
}
}
@@ -72,9 +72,48 @@ func TestMarshalBodyAllocatesTwoBodies(t *testing.T) {
return m.TotalAlloc - before, n
}
stdAlloc, n := peak(func() []byte { b, _ := json.Marshal(body); return b })
- ownAlloc, _ := peak(func() []byte { b, _ := marshalBody(body); return b })
+ ownAlloc, _ := peak(func() []byte { b, _ := marshalBody(body, ""); return b })
t.Logf("%d MiB body: json.Marshal allocated %d MiB, marshalBody %d MiB", n>>20, stdAlloc>>20, ownAlloc>>20)
if ownAlloc > uint64(n)*5/2 {
t.Errorf("marshalBody allocated %d MiB for a %d MiB body, want at most 2.5x", ownAlloc>>20, n>>20)
}
}
+
+// spentProbe is an item of a list that, as it is written, looks whether
+// the item before it was let go.
+type spentProbe struct {
+ list *[]*spentProbe
+ i int
+ held *[]int
+}
+
+func (p *spentProbe) MarshalJSON() ([]byte, error) {
+ if p.i > 0 && (*p.list)[p.i-1] != nil {
+ *p.held = append(*p.held, p.i-1)
+ }
+ return []byte(`1`), nil
+}
+
+// The conversation a builder made for the body is let go an item at a
+// time as the body fills, not held beside it to the end: built for Chat,
+// a 45 MiB Codex conversation's messages, strings joined from its parts
+// among them, were a copy more at the body's peak (#4004). A list not
+// named spent is left as it was.
+func TestMarshalBodyLetsSpentGo(t *testing.T) {
+ var held []int
+ list := make([]*spentProbe, 5)
+ for i := range list {
+ list[i] = &spentProbe{list: &list, i: i, held: &held}
+ }
+ other := []string{"a", "b"}
+ got, err := marshalBody(map[string]any{"messages": list, "other": other}, "messages")
+ if err != nil || string(got) != `{"messages":[1,1,1,1,1],"other":["a","b"]}` {
+ t.Fatalf("wrote %s (%v)", got, err)
+ }
+ if len(held) > 0 {
+ t.Errorf("items %v were still held when the next was written", held)
+ }
+ if other[0] != "a" || other[1] != "b" {
+ t.Errorf("a list not spent was changed: %q", other)
+ }
+}
diff --git a/internal/gateway/responses.go b/internal/gateway/responses.go
index b14549b13..b056cd80e 100644
--- a/internal/gateway/responses.go
+++ b/internal/gateway/responses.go
@@ -1357,7 +1357,7 @@ func buildResponses(r *Request, model, host string, rejectTemp bool) []byte {
if len(include) > 0 {
out["include"] = include
}
- b, _ := marshalBody(out)
+ b, _ := marshalBody(out, "input")
// a call the client left unanswered (a turn it interrupted) gets its
// synthetic result, as buildChat's pairToolMessages and
// buildAnthropic's pairToolBlocks give one: the Responses API refuses