Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a translated body lets its built conversation go as it fills (#4004)
yetone8eb7f5565bda
diff --git a/internal/gateway/anthropic.go b/internal/gateway/anthropic.goindex 4497f4e6d..a445f15ff 100644--- a/internal/gateway/anthropic.go+++ b/internal/gateway/anthropic.go@@ -1099,7 +1099,7 @@ func anthropicBody(r *Request, model, host string, replay bool) []byte {out["tool_choice"] = tc}}- b, _ := marshalBody(out)+ b, _ := marshalBody(out, "messages")return b}diff --git a/internal/gateway/chat.go b/internal/gateway/chat.goindex 7edff257c..0fca9b3ef 100644--- a/internal/gateway/chat.go+++ b/internal/gateway/chat.go@@ -585,7 +585,7 @@ func buildChat(r *Request, model, host string, rejectTemp bool) []byte {// OpenRouter's own search, for any of its modelsout["plugins"] = []map[string]any{{"id": "web"}}}- b, _ := marshalBody(out)+ b, _ := marshalBody(out, "messages")return b}diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.goindex 28e29b504..21f19715e 100644--- a/internal/gateway/fastjson.go+++ b/internal/gateway/fastjson.go@@ -1447,16 +1447,27 @@ func getIn(body []byte, path string) gjson.Result {// 316 MiB, the buffers it outgrew being garbage until the next GC (#4004,// jihuayu's profile on a 2 GiB server). Here no buffer outgrows the// largest item. A value json.Marshal refuses gets json.Marshal's answer.-func marshalBody(out map[string]any) ([]byte, error) {+//+// spent names the list the builder made for this body alone (its+// messages, its input items): each of its items is let go once written,+// so the built conversation, strings joined from the request's parts+// among it, is garbage while the body fills instead of held beside it to+// the end (#4004). A value json.Marshal refuses is refused still, and the+// list is then left part spent.+func marshalBody(out map[string]any, spent string) ([]byte, error) {keys := make([]string, 0, len(out))for k := range out {+ // a key json.Marshal writes another way, before anything is spent+ if _, ok := appendJSONKey(nil, k, true); !ok {+ return json.Marshal(out)+ }keys = append(keys, k)}slices.Sort(keys)vals := make([][]json.RawMessage, len(keys))n := 2for i, k := range keys {- items, ok := marshalItems(out[k])+ items, ok := marshalItems(out[k], k == spent)if !ok {return json.Marshal(out)}@@ -1507,8 +1518,8 @@ func isSplitList(v any) bool {// marshalItems is v as json.Marshal writes it: its items each on their// own for a list isSplitList takes, or one piece; false when json.Marshal-// refuses any of it.-func marshalItems(v any) ([]json.RawMessage, bool) {+// refuses any of it. spend sets each item written to its zero value.+func marshalItems(v any, spend bool) ([]json.RawMessage, bool) {if !isSplitList(v) {b, err := json.Marshal(v)return []json.RawMessage{b}, err == nil@@ -1523,6 +1534,9 @@ func marshalItems(v any) ([]json.RawMessage, bool) {return nil, false}items[i] = b+ if spend {+ rv.Index(i).SetZero()+ }}return items, true}diff --git a/internal/gateway/gemini.go b/internal/gateway/gemini.goindex 2c031803a..03c3ef542 100644--- a/internal/gateway/gemini.go+++ b/internal/gateway/gemini.go@@ -116,7 +116,7 @@ func buildGemini(r *Request, model string) ([]byte, error) {return nil, err}req["model"] = model- return marshalBody(req)+ return marshalBody(req, "")}// geminiRequest is the request inside the Code Assist envelope thatdiff --git a/internal/gateway/marshal_body_test.go b/internal/gateway/marshal_body_test.goindex 5b3b11d2b..7428028db 100644--- a/internal/gateway/marshal_body_test.go+++ b/internal/gateway/marshal_body_test.go@@ -33,7 +33,7 @@ func TestMarshalBodyIsJSONMarshal(t *testing.T) {}for i, b := range bodies {want, werr := json.Marshal(b)- got, gerr := marshalBody(b)+ got, gerr := marshalBody(b, "messages")if (werr == nil) != (gerr == nil) || !bytes.Equal(got, want) {t.Errorf("body %d:\n got %s (%v)\nwant %s (%v)", i, got, gerr, want, werr)}@@ -43,7 +43,7 @@ func TestMarshalBodyIsJSONMarshal(t *testing.T) {{"x": math.Inf(1)},{"ch": []any{"a", make(chan int)}},} {- if _, err := marshalBody(bad); err == nil {+ if _, err := marshalBody(bad, ""); err == nil {t.Errorf("marshalBody(%v) took what json.Marshal refuses", bad)}}@@ -72,9 +72,48 @@ func TestMarshalBodyAllocatesTwoBodies(t *testing.T) {return m.TotalAlloc - before, n}stdAlloc, n := peak(func() []byte { b, _ := json.Marshal(body); return b })- ownAlloc, _ := peak(func() []byte { b, _ := marshalBody(body); return b })+ ownAlloc, _ := peak(func() []byte { b, _ := marshalBody(body, ""); return b })t.Logf("%d MiB body: json.Marshal allocated %d MiB, marshalBody %d MiB", n>>20, stdAlloc>>20, ownAlloc>>20)if ownAlloc > uint64(n)*5/2 {t.Errorf("marshalBody allocated %d MiB for a %d MiB body, want at most 2.5x", ownAlloc>>20, n>>20)}}++// spentProbe is an item of a list that, as it is written, looks whether+// the item before it was let go.+type spentProbe struct {+ list *[]*spentProbe+ i int+ held *[]int+}++func (p *spentProbe) MarshalJSON() ([]byte, error) {+ if p.i > 0 && (*p.list)[p.i-1] != nil {+ *p.held = append(*p.held, p.i-1)+ }+ return []byte(`1`), nil+}++// The conversation a builder made for the body is let go an item at a+// time as the body fills, not held beside it to the end: built for Chat,+// a 45 MiB Codex conversation's messages, strings joined from its parts+// among them, were a copy more at the body's peak (#4004). A list not+// named spent is left as it was.+func TestMarshalBodyLetsSpentGo(t *testing.T) {+ var held []int+ list := make([]*spentProbe, 5)+ for i := range list {+ list[i] = &spentProbe{list: &list, i: i, held: &held}+ }+ other := []string{"a", "b"}+ got, err := marshalBody(map[string]any{"messages": list, "other": other}, "messages")+ if err != nil || string(got) != `{"messages":[1,1,1,1,1],"other":["a","b"]}` {+ t.Fatalf("wrote %s (%v)", got, err)+ }+ if len(held) > 0 {+ t.Errorf("items %v were still held when the next was written", held)+ }+ if other[0] != "a" || other[1] != "b" {+ t.Errorf("a list not spent was changed: %q", other)+ }+}diff --git a/internal/gateway/responses.go b/internal/gateway/responses.goindex b14549b13..b056cd80e 100644--- a/internal/gateway/responses.go+++ b/internal/gateway/responses.go@@ -1357,7 +1357,7 @@ func buildResponses(r *Request, model, host string, rejectTemp bool) []byte {if len(include) > 0 {out["include"] = include}- b, _ := marshalBody(out)+ b, _ := marshalBody(out, "input")// a call the client left unanswered (a turn it interrupted) gets its// synthetic result, as buildChat's pairToolMessages and// buildAnthropic's pairToolBlocks give one: the Responses API refuses