Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a translated body is written a message at a time (#4004)
yetone0b99b8fbe58d
diff --git a/internal/gateway/anthropic.go b/internal/gateway/anthropic.goindex 0f3101705..4497f4e6d 100644--- a/internal/gateway/anthropic.go+++ b/internal/gateway/anthropic.go@@ -1099,7 +1099,7 @@ func anthropicBody(r *Request, model, host string, replay bool) []byte {out["tool_choice"] = tc}}- b, _ := json.Marshal(out)+ b, _ := marshalBody(out)return b}diff --git a/internal/gateway/chat.go b/internal/gateway/chat.goindex ba3fffb0d..3a0759b82 100644--- a/internal/gateway/chat.go+++ b/internal/gateway/chat.go@@ -584,7 +584,7 @@ func buildChat(r *Request, model, host string, rejectTemp bool) []byte {// OpenRouter's own search, for any of its modelsout["plugins"] = []map[string]any{{"id": "web"}}}- b, _ := json.Marshal(out)+ b, _ := marshalBody(out)return b}diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.goindex a39c59880..01ca843cb 100644--- a/internal/gateway/fastjson.go+++ b/internal/gateway/fastjson.go@@ -1426,3 +1426,91 @@ func appendHTMLEscaped(dst, s []byte) []byte {}return append(dst, s[start:]...)}++// marshalBody is json.Marshal of a request body being built, byte for+// byte, its lists (the messages, the input items) written an item at a+// time into a buffer of the size they come to. json.Marshal writes the+// whole body into one buffer that doubles as it fills, then copies it+// out: a 45 MB Codex conversation built for Chat took the heap from 45 to+// 316 MiB, the buffers it outgrew being garbage until the next GC (#4004,+// jihuayu's profile on a 2 GiB server). Here no buffer outgrows the+// largest item. A value json.Marshal refuses gets json.Marshal's answer.+func marshalBody(out map[string]any) ([]byte, error) {+ keys := make([]string, 0, len(out))+ for k := range out {+ keys = append(keys, k)+ }+ slices.Sort(keys)+ vals := make([][]json.RawMessage, len(keys))+ n := 2+ for i, k := range keys {+ items, ok := marshalItems(out[k])+ if !ok {+ return json.Marshal(out)+ }+ vals[i] = items+ n += len(k) + 4+ for _, it := range items {+ n += len(it) + 1+ }+ }+ dst := make([]byte, 0, n)+ dst = append(dst, '{')+ for i, k := range keys {+ if i > 0 {+ dst = append(dst, ',')+ }+ var ok bool+ if dst, ok = appendJSONKey(dst, k, true); !ok {+ return json.Marshal(out)+ }+ dst = append(dst, ':')+ items := vals[i]+ vals[i] = nil+ if !isSplitList(out[k]) {+ dst = append(dst, items[0]...)+ continue+ }+ dst = append(dst, '[')+ for j, it := range items {+ if j > 0 {+ dst = append(dst, ',')+ }+ dst = append(dst, it...)+ }+ dst = append(dst, ']')+ }+ return append(dst, '}'), nil+}++// isSplitList says whether marshalItems writes v an item at a time: a+// slice of two or more that isn't bytes and doesn't write itself.+func isSplitList(v any) bool {+ if _, ok := v.(json.Marshaler); ok {+ return false+ }+ rv := reflect.ValueOf(v)+ return rv.Kind() == reflect.Slice && rv.Type().Elem().Kind() != reflect.Uint8 && rv.Len() > 1+}++// marshalItems is v as json.Marshal writes it: its items each on their+// own for a list isSplitList takes, or one piece; false when json.Marshal+// refuses any of it.+func marshalItems(v any) ([]json.RawMessage, bool) {+ if !isSplitList(v) {+ b, err := json.Marshal(v)+ return []json.RawMessage{b}, err == nil+ }+ rv := reflect.ValueOf(v)+ items := make([]json.RawMessage, rv.Len())+ for i := range items {+ // by its address, as json.Marshal reaches a slice's items: a+ // MarshalJSON on the pointer is called+ b, err := json.Marshal(rv.Index(i).Addr().Interface())+ if err != nil {+ return nil, false+ }+ items[i] = b+ }+ return items, true+}diff --git a/internal/gateway/gemini.go b/internal/gateway/gemini.goindex b52fd9f44..2c031803a 100644--- a/internal/gateway/gemini.go+++ b/internal/gateway/gemini.go@@ -116,7 +116,7 @@ func buildGemini(r *Request, model string) ([]byte, error) {return nil, err}req["model"] = model- return json.Marshal(req)+ return marshalBody(req)}// geminiRequest is the request inside the Code Assist envelope thatdiff --git a/internal/gateway/marshal_body_test.go b/internal/gateway/marshal_body_test.gonew file mode 100644index 000000000..5ceb81270--- /dev/null+++ b/internal/gateway/marshal_body_test.go@@ -0,0 +1,77 @@+package gateway++import (+ "bytes"+ "encoding/json"+ "math"+ "runtime"+ "strings"+ "testing"+)++type ptrMarshaler struct{ n int }++func (p *ptrMarshaler) MarshalJSON() ([]byte, error) { return []byte(`"ptr"`), nil }++type valMarshaler []int++func (valMarshaler) MarshalJSON() ([]byte, error) { return []byte(`"whole"`), nil }++// marshalBody writes what json.Marshal writes, every kind of value a+// builder puts in a body included.+func TestMarshalBodyIsJSONMarshal(t *testing.T) {+ var nilList []any+ bodies := []map[string]any{+ {},+ {"model": "m", "stream": true, "max_tokens": 10, "temperature": 0.5},+ {"messages": []any{map[string]any{"role": "user", "content": "<a&b> \u2028 \u2029 \x01 é"}, map[string]any{"role": "assistant", "content": nil}}},+ {"messages": []map[string]any{{"role": "user"}, {"role": "system", "content": []any{"x", 1}}}},+ {"one": []any{"only"}, "none": []any{}, "nil": nilList, "nilv": nil},+ {"bytes": []byte("raw bytes"), "raw": json.RawMessage(`{"a":1}`), "raws": []json.RawMessage{json.RawMessage(`1`), json.RawMessage(`{"b":2}`)}},+ {"ptrs": []ptrMarshaler{{1}, {2}}, "whole": valMarshaler{1, 2, 3}, "strs": []string{"a", "<b>"}},+ {"<k>": 1, "a\u2028": 2, "z": []int{1, 2, 3}},+ }+ for i, b := range bodies {+ want, werr := json.Marshal(b)+ got, gerr := marshalBody(b)+ if (werr == nil) != (gerr == nil) || !bytes.Equal(got, want) {+ t.Errorf("body %d:\n got %s (%v)\nwant %s (%v)", i, got, gerr, want, werr)+ }+ }+ for _, bad := range []map[string]any{+ {"messages": []any{1, math.NaN()}},+ {"x": math.Inf(1)},+ {"ch": []any{"a", make(chan int)}},+ } {+ if _, err := marshalBody(bad); err == nil {+ t.Errorf("marshalBody(%v) took what json.Marshal refuses", bad)+ }+ }+}++// A 45 MB conversation is built without the heap growing past a body+// and its items: json.Marshal's doubling buffer took it from 45 to 316+// MiB (#4004).+func TestMarshalBodyAllocatesTwoBodies(t *testing.T) {+ big := strings.Repeat("tool output line\n", 1<<14)+ msgs := make([]any, 0, 170)+ for range 170 {+ msgs = append(msgs, map[string]any{"role": "tool", "tool_call_id": "c", "content": big})+ }+ body := map[string]any{"model": "m", "messages": msgs}+ peak := func(f func() []byte) (uint64, int) {+ runtime.GC()+ var m runtime.MemStats+ runtime.ReadMemStats(&m)+ before := m.TotalAlloc+ n := len(f())+ runtime.ReadMemStats(&m)+ return m.TotalAlloc - before, n+ }+ stdAlloc, n := peak(func() []byte { b, _ := json.Marshal(body); return b })+ ownAlloc, _ := peak(func() []byte { b, _ := marshalBody(body); return b })+ t.Logf("%d MiB body: json.Marshal allocated %d MiB, marshalBody %d MiB", n>>20, stdAlloc>>20, ownAlloc>>20)+ if ownAlloc > uint64(n)*5/2 {+ t.Errorf("marshalBody allocated %d MiB for a %d MiB body, want at most 2.5x", ownAlloc>>20, n>>20)+ }+}diff --git a/internal/gateway/responses.go b/internal/gateway/responses.goindex c7d07b7e3..b14549b13 100644--- a/internal/gateway/responses.go+++ b/internal/gateway/responses.go@@ -1357,7 +1357,7 @@ func buildResponses(r *Request, model, host string, rejectTemp bool) []byte {if len(include) > 0 {out["include"] = include}- b, _ := json.Marshal(out)+ b, _ := marshalBody(out)// a call the client left unanswered (a turn it interrupted) gets its// synthetic result, as buildChat's pairToolMessages and// buildAnthropic's pairToolBlocks give one: the Responses API refuses