Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a translated body is written a message at a time (#4004)

yetone0b99b8fbe58d
diff --git a/internal/gateway/anthropic.go b/internal/gateway/anthropic.go
index 0f3101705..4497f4e6d 100644
--- a/internal/gateway/anthropic.go
+++ b/internal/gateway/anthropic.go
@@ -1099,7 +1099,7 @@ func anthropicBody(r *Request, model, host string, replay bool) []byte {
out["tool_choice"] = tc
}
}
- b, _ := json.Marshal(out)
+ b, _ := marshalBody(out)
return b
}
diff --git a/internal/gateway/chat.go b/internal/gateway/chat.go
index ba3fffb0d..3a0759b82 100644
--- a/internal/gateway/chat.go
+++ b/internal/gateway/chat.go
@@ -584,7 +584,7 @@ func buildChat(r *Request, model, host string, rejectTemp bool) []byte {
// OpenRouter's own search, for any of its models
out["plugins"] = []map[string]any{{"id": "web"}}
}
- b, _ := json.Marshal(out)
+ b, _ := marshalBody(out)
return b
}
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.go
index a39c59880..01ca843cb 100644
--- a/internal/gateway/fastjson.go
+++ b/internal/gateway/fastjson.go
@@ -1426,3 +1426,91 @@ func appendHTMLEscaped(dst, s []byte) []byte {
}
return append(dst, s[start:]...)
}
+
+// marshalBody is json.Marshal of a request body being built, byte for
+// byte, its lists (the messages, the input items) written an item at a
+// time into a buffer of the size they come to. json.Marshal writes the
+// whole body into one buffer that doubles as it fills, then copies it
+// out: a 45 MB Codex conversation built for Chat took the heap from 45 to
+// 316 MiB, the buffers it outgrew being garbage until the next GC (#4004,
+// jihuayu's profile on a 2 GiB server). Here no buffer outgrows the
+// largest item. A value json.Marshal refuses gets json.Marshal's answer.
+func marshalBody(out map[string]any) ([]byte, error) {
+ keys := make([]string, 0, len(out))
+ for k := range out {
+ keys = append(keys, k)
+ }
+ slices.Sort(keys)
+ vals := make([][]json.RawMessage, len(keys))
+ n := 2
+ for i, k := range keys {
+ items, ok := marshalItems(out[k])
+ if !ok {
+ return json.Marshal(out)
+ }
+ vals[i] = items
+ n += len(k) + 4
+ for _, it := range items {
+ n += len(it) + 1
+ }
+ }
+ dst := make([]byte, 0, n)
+ dst = append(dst, '{')
+ for i, k := range keys {
+ if i > 0 {
+ dst = append(dst, ',')
+ }
+ var ok bool
+ if dst, ok = appendJSONKey(dst, k, true); !ok {
+ return json.Marshal(out)
+ }
+ dst = append(dst, ':')
+ items := vals[i]
+ vals[i] = nil
+ if !isSplitList(out[k]) {
+ dst = append(dst, items[0]...)
+ continue
+ }
+ dst = append(dst, '[')
+ for j, it := range items {
+ if j > 0 {
+ dst = append(dst, ',')
+ }
+ dst = append(dst, it...)
+ }
+ dst = append(dst, ']')
+ }
+ return append(dst, '}'), nil
+}
+
+// isSplitList says whether marshalItems writes v an item at a time: a
+// slice of two or more that isn't bytes and doesn't write itself.
+func isSplitList(v any) bool {
+ if _, ok := v.(json.Marshaler); ok {
+ return false
+ }
+ rv := reflect.ValueOf(v)
+ return rv.Kind() == reflect.Slice && rv.Type().Elem().Kind() != reflect.Uint8 && rv.Len() > 1
+}
+
+// marshalItems is v as json.Marshal writes it: its items each on their
+// own for a list isSplitList takes, or one piece; false when json.Marshal
+// refuses any of it.
+func marshalItems(v any) ([]json.RawMessage, bool) {
+ if !isSplitList(v) {
+ b, err := json.Marshal(v)
+ return []json.RawMessage{b}, err == nil
+ }
+ rv := reflect.ValueOf(v)
+ items := make([]json.RawMessage, rv.Len())
+ for i := range items {
+ // by its address, as json.Marshal reaches a slice's items: a
+ // MarshalJSON on the pointer is called
+ b, err := json.Marshal(rv.Index(i).Addr().Interface())
+ if err != nil {
+ return nil, false
+ }
+ items[i] = b
+ }
+ return items, true
+}
diff --git a/internal/gateway/gemini.go b/internal/gateway/gemini.go
index b52fd9f44..2c031803a 100644
--- a/internal/gateway/gemini.go
+++ b/internal/gateway/gemini.go
@@ -116,7 +116,7 @@ func buildGemini(r *Request, model string) ([]byte, error) {
return nil, err
}
req["model"] = model
- return json.Marshal(req)
+ return marshalBody(req)
}
// geminiRequest is the request inside the Code Assist envelope that
diff --git a/internal/gateway/marshal_body_test.go b/internal/gateway/marshal_body_test.go
new file mode 100644
index 000000000..5ceb81270
--- /dev/null
+++ b/internal/gateway/marshal_body_test.go
@@ -0,0 +1,77 @@
+package gateway
+
+import (
+ "bytes"
+ "encoding/json"
+ "math"
+ "runtime"
+ "strings"
+ "testing"
+)
+
+type ptrMarshaler struct{ n int }
+
+func (p *ptrMarshaler) MarshalJSON() ([]byte, error) { return []byte(`"ptr"`), nil }
+
+type valMarshaler []int
+
+func (valMarshaler) MarshalJSON() ([]byte, error) { return []byte(`"whole"`), nil }
+
+// marshalBody writes what json.Marshal writes, every kind of value a
+// builder puts in a body included.
+func TestMarshalBodyIsJSONMarshal(t *testing.T) {
+ var nilList []any
+ bodies := []map[string]any{
+ {},
+ {"model": "m", "stream": true, "max_tokens": 10, "temperature": 0.5},
+ {"messages": []any{map[string]any{"role": "user", "content": "<a&b> \u2028 \u2029 \x01 é"}, map[string]any{"role": "assistant", "content": nil}}},
+ {"messages": []map[string]any{{"role": "user"}, {"role": "system", "content": []any{"x", 1}}}},
+ {"one": []any{"only"}, "none": []any{}, "nil": nilList, "nilv": nil},
+ {"bytes": []byte("raw bytes"), "raw": json.RawMessage(`{"a":1}`), "raws": []json.RawMessage{json.RawMessage(`1`), json.RawMessage(`{"b":2}`)}},
+ {"ptrs": []ptrMarshaler{{1}, {2}}, "whole": valMarshaler{1, 2, 3}, "strs": []string{"a", "<b>"}},
+ {"<k>": 1, "a\u2028": 2, "z": []int{1, 2, 3}},
+ }
+ for i, b := range bodies {
+ want, werr := json.Marshal(b)
+ got, gerr := marshalBody(b)
+ if (werr == nil) != (gerr == nil) || !bytes.Equal(got, want) {
+ t.Errorf("body %d:\n got %s (%v)\nwant %s (%v)", i, got, gerr, want, werr)
+ }
+ }
+ for _, bad := range []map[string]any{
+ {"messages": []any{1, math.NaN()}},
+ {"x": math.Inf(1)},
+ {"ch": []any{"a", make(chan int)}},
+ } {
+ if _, err := marshalBody(bad); err == nil {
+ t.Errorf("marshalBody(%v) took what json.Marshal refuses", bad)
+ }
+ }
+}
+
+// A 45 MB conversation is built without the heap growing past a body
+// and its items: json.Marshal's doubling buffer took it from 45 to 316
+// MiB (#4004).
+func TestMarshalBodyAllocatesTwoBodies(t *testing.T) {
+ big := strings.Repeat("tool output line\n", 1<<14)
+ msgs := make([]any, 0, 170)
+ for range 170 {
+ msgs = append(msgs, map[string]any{"role": "tool", "tool_call_id": "c", "content": big})
+ }
+ body := map[string]any{"model": "m", "messages": msgs}
+ peak := func(f func() []byte) (uint64, int) {
+ runtime.GC()
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ before := m.TotalAlloc
+ n := len(f())
+ runtime.ReadMemStats(&m)
+ return m.TotalAlloc - before, n
+ }
+ stdAlloc, n := peak(func() []byte { b, _ := json.Marshal(body); return b })
+ ownAlloc, _ := peak(func() []byte { b, _ := marshalBody(body); return b })
+ t.Logf("%d MiB body: json.Marshal allocated %d MiB, marshalBody %d MiB", n>>20, stdAlloc>>20, ownAlloc>>20)
+ if ownAlloc > uint64(n)*5/2 {
+ t.Errorf("marshalBody allocated %d MiB for a %d MiB body, want at most 2.5x", ownAlloc>>20, n>>20)
+ }
+}
diff --git a/internal/gateway/responses.go b/internal/gateway/responses.go
index c7d07b7e3..b14549b13 100644
--- a/internal/gateway/responses.go
+++ b/internal/gateway/responses.go
@@ -1357,7 +1357,7 @@ func buildResponses(r *Request, model, host string, rejectTemp bool) []byte {
if len(include) > 0 {
out["include"] = include
}
- b, _ := json.Marshal(out)
+ b, _ := marshalBody(out)
// a call the client left unanswered (a turn it interrupted) gets its
// synthetic result, as buildChat's pairToolMessages and
// buildAnthropic's pairToolBlocks give one: the Responses API refuses