Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a large translated body is written into itself, its items not kept beside it (#4004)
yetoneddf31ca305d3
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.goindex 21f19715e..35f64a9eb 100644--- a/internal/gateway/fastjson.go+++ b/internal/gateway/fastjson.go@@ -1454,6 +1454,12 @@ func getIn(body []byte, path string) gjson.Result {// among it, is garbage while the body fills instead of held beside it to// the end (#4004). A value json.Marshal refuses is refused still, and the// list is then left part spent.+//+// Past bodyKept of them, the written items aren't kept for the body+// either: each is measured, its bytes written to no buffer, and written+// again into the body once that is made. Kept, they were a copy of the+// conversation beside the body at its peak; written twice costs a large+// body's encoding once more, and a body under bodyKept nothing.func marshalBody(out map[string]any, spent string) ([]byte, error) {keys := make([]string, 0, len(out))for k := range out {@@ -1465,17 +1471,15 @@ func marshalBody(out map[string]any, spent string) ([]byte, error) {}slices.Sort(keys)vals := make([][]json.RawMessage, len(keys))+ kept := bodyKeptn := 2for i, k := range keys {- items, ok := marshalItems(out[k], k == spent)+ items, sizes, ok := marshalItems(out[k], k == spent, &kept)if !ok {return json.Marshal(out)}vals[i] = items- n += len(k) + 4- for _, it := range items {- n += len(it) + 1- }+ n += len(k) + 4 + len(items) + sizes}dst := make([]byte, 0, n)dst = append(dst, '{')@@ -1495,11 +1499,24 @@ func marshalBody(out map[string]any, spent string) ([]byte, error) {continue}dst = append(dst, '[')+ list := reflect.ValueOf(out[k])for j, it := range items {if j > 0 {dst = append(dst, ',')}- dst = append(dst, it...)+ if it != nil {+ dst = append(dst, it...)+ continue+ }+ // measured only: written now, straight into the body+ w := appendWriter{dst}+ if err := json.NewEncoder(&w).Encode(list.Index(j).Addr().Interface()); err != nil {+ return nil, err+ }+ dst = w.b[:len(w.b)-1] // Encode's newline+ if k == spent {+ list.Index(j).SetZero()+ }}dst = append(dst, ']')}@@ -1518,25 +1535,57 @@ func isSplitList(v any) bool {// marshalItems is v as json.Marshal writes it: its items each on their// own for a list isSplitList takes, or one piece; false when json.Marshal-// refuses any of it. spend sets each item written to its zero value.-func marshalItems(v any, spend bool) ([]json.RawMessage, bool) {+// refuses any of it, and the bytes they come to. An item is kept while+// *kept, the bytes left to keep, is above 0; past it an item is only+// measured, nil among the items, and left as it was. spend sets each item+// kept to its zero value.+func marshalItems(v any, spend bool, kept *int) ([]json.RawMessage, int, bool) {if !isSplitList(v) {b, err := json.Marshal(v)- return []json.RawMessage{b}, err == nil+ *kept -= len(b)+ return []json.RawMessage{b}, len(b), err == nil}rv := reflect.ValueOf(v)items := make([]json.RawMessage, rv.Len())+ var count countWriter+ measure := json.NewEncoder(&count)+ size := 0for i := range items {// by its address, as json.Marshal reaches a slice's items: a// MarshalJSON on the pointer is called- b, err := json.Marshal(rv.Index(i).Addr().Interface())+ it := rv.Index(i).Addr().Interface()+ if *kept <= 0 {+ before := count.n+ if measure.Encode(it) != nil {+ return nil, 0, false+ }+ size += count.n - before - 1 // Encode's newline+ continue+ }+ b, err := json.Marshal(it)if err != nil {- return nil, false+ return nil, 0, false}items[i] = b+ size += len(b)+ *kept -= len(b)if spend {rv.Index(i).SetZero()}}- return items, true+ return items, size, true}++// bodyKept is how many bytes of a body's written items marshalBody keeps+// to put in the body; past it they are measured and written again.+var bodyKept = 4 << 20++// countWriter counts what is written to it.+type countWriter struct{ n int }++func (w *countWriter) Write(b []byte) (int, error) { w.n += len(b); return len(b), nil }++// appendWriter appends what is written to it to b.+type appendWriter struct{ b []byte }++func (w *appendWriter) Write(p []byte) (int, error) { w.b = append(w.b, p...); return len(p), nil }diff --git a/internal/gateway/marshal_body_test.go b/internal/gateway/marshal_body_test.goindex 7428028db..5444d1f7c 100644--- a/internal/gateway/marshal_body_test.go+++ b/internal/gateway/marshal_body_test.go@@ -21,37 +21,48 @@ func (valMarshaler) MarshalJSON() ([]byte, error) { return []byte(`"whole"`), ni// builder puts in a body included.func TestMarshalBodyIsJSONMarshal(t *testing.T) {var nilList []any- bodies := []map[string]any{- {},- {"model": "m", "stream": true, "max_tokens": 10, "temperature": 0.5},- {"messages": []any{map[string]any{"role": "user", "content": "<a&b> \u2028 \u2029 \x01 é"}, map[string]any{"role": "assistant", "content": nil}}},- {"messages": []map[string]any{{"role": "user"}, {"role": "system", "content": []any{"x", 1}}}},- {"one": []any{"only"}, "none": []any{}, "nil": nilList, "nilv": nil},- {"bytes": []byte("raw bytes"), "raw": json.RawMessage(`{"a":1}`), "raws": []json.RawMessage{json.RawMessage(`1`), json.RawMessage(`{"b":2}`)}},- {"ptrs": []ptrMarshaler{{1}, {2}}, "whole": valMarshaler{1, 2, 3}, "strs": []string{"a", "<b>"}},- {"<k>": 1, "a\u2028": 2, "z": []int{1, 2, 3}},+ bodies := func() []map[string]any {+ return []map[string]any{+ {},+ {"model": "m", "stream": true, "max_tokens": 10, "temperature": 0.5},+ {"messages": []any{map[string]any{"role": "user", "content": "<a&b> \u2028 \u2029 \x01 é"}, map[string]any{"role": "assistant", "content": nil}}},+ {"messages": []map[string]any{{"role": "user"}, {"role": "system", "content": []any{"x", 1}}}},+ {"one": []any{"only"}, "none": []any{}, "nil": nilList, "nilv": nil},+ {"bytes": []byte("raw bytes"), "raw": json.RawMessage(`{"a":1}`), "raws": []json.RawMessage{json.RawMessage(`1`), json.RawMessage(`{"b":2}`)}},+ {"ptrs": []ptrMarshaler{{1}, {2}}, "whole": valMarshaler{1, 2, 3}, "strs": []string{"a", "<b>"}},+ {"<k>": 1, "a\u2028": 2, "z": []int{1, 2, 3}},+ }}- for i, b := range bodies {- want, werr := json.Marshal(b)- got, gerr := marshalBody(b, "messages")- if (werr == nil) != (gerr == nil) || !bytes.Equal(got, want) {- t.Errorf("body %d:\n got %s (%v)\nwant %s (%v)", i, got, gerr, want, werr)+ // kept whole, measured from the first item on, and from the second+ t.Cleanup(func() { bodyKept = 4 << 20 })+ for _, kept := range []int{4 << 20, 0, 1} {+ bodyKept = kept+ for i, b := range bodies() { // fresh: a spent list is zeroed+ want, werr := json.Marshal(b)+ got, gerr := marshalBody(b, "messages")+ if (werr == nil) != (gerr == nil) || !bytes.Equal(got, want) {+ t.Errorf("kept %d, body %d:\n got %s (%v)\nwant %s (%v)", kept, i, got, gerr, want, werr)+ }}}- for _, bad := range []map[string]any{- {"messages": []any{1, math.NaN()}},- {"x": math.Inf(1)},- {"ch": []any{"a", make(chan int)}},- } {- if _, err := marshalBody(bad, ""); err == nil {- t.Errorf("marshalBody(%v) took what json.Marshal refuses", bad)+ for _, kept := range []int{4 << 20, 0} {+ bodyKept = kept+ for _, bad := range []map[string]any{+ {"messages": []any{1, math.NaN()}},+ {"x": math.Inf(1)},+ {"ch": []any{"a", make(chan int)}},+ } {+ if _, err := marshalBody(bad, ""); err == nil {+ t.Errorf("kept %d: marshalBody(%v) took what json.Marshal refuses", kept, bad)+ }}}}-// A 45 MB conversation is built without the heap growing past a body-// and its items: json.Marshal's doubling buffer took it from 45 to 316-// MiB (#4004).+// A 45 MB conversation is built without the heap growing past the body+// and a few MiB: json.Marshal's doubling buffer took it from 45 to 316+// MiB, and keeping every written item for the body was a copy more+// (#4004).func TestMarshalBodyAllocatesTwoBodies(t *testing.T) {if raceOn {t.Skip("under -race sync.Pool drops what json.Marshal's encoders keep, so each item allocates afresh")@@ -74,8 +85,10 @@ func TestMarshalBodyAllocatesTwoBodies(t *testing.T) {stdAlloc, n := peak(func() []byte { b, _ := json.Marshal(body); return b })ownAlloc, _ := peak(func() []byte { b, _ := marshalBody(body, ""); return b })t.Logf("%d MiB body: json.Marshal allocated %d MiB, marshalBody %d MiB", n>>20, stdAlloc>>20, ownAlloc>>20)- if ownAlloc > uint64(n)*5/2 {- t.Errorf("marshalBody allocated %d MiB for a %d MiB body, want at most 2.5x", ownAlloc>>20, n>>20)+ // the body, the items kept for it and the encoder's buffer: no copy+ // of every item beside the body (#4004)+ if ownAlloc > uint64(n)*3/2 {+ t.Errorf("marshalBody allocated %d MiB for a %d MiB body, want at most 1.5x", ownAlloc>>20, n>>20)}}