Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a large translated body is written into itself, its items not kept beside it (#4004)

yetoneddf31ca305d3
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.go
index 21f19715e..35f64a9eb 100644
--- a/internal/gateway/fastjson.go
+++ b/internal/gateway/fastjson.go
@@ -1454,6 +1454,12 @@ func getIn(body []byte, path string) gjson.Result {
// among it, is garbage while the body fills instead of held beside it to
// the end (#4004). A value json.Marshal refuses is refused still, and the
// list is then left part spent.
+//
+// Past bodyKept of them, the written items aren't kept for the body
+// either: each is measured, its bytes written to no buffer, and written
+// again into the body once that is made. Kept, they were a copy of the
+// conversation beside the body at its peak; written twice costs a large
+// body's encoding once more, and a body under bodyKept nothing.
func marshalBody(out map[string]any, spent string) ([]byte, error) {
keys := make([]string, 0, len(out))
for k := range out {
@@ -1465,17 +1471,15 @@ func marshalBody(out map[string]any, spent string) ([]byte, error) {
}
slices.Sort(keys)
vals := make([][]json.RawMessage, len(keys))
+ kept := bodyKept
n := 2
for i, k := range keys {
- items, ok := marshalItems(out[k], k == spent)
+ items, sizes, ok := marshalItems(out[k], k == spent, &kept)
if !ok {
return json.Marshal(out)
}
vals[i] = items
- n += len(k) + 4
- for _, it := range items {
- n += len(it) + 1
- }
+ n += len(k) + 4 + len(items) + sizes
}
dst := make([]byte, 0, n)
dst = append(dst, '{')
@@ -1495,11 +1499,24 @@ func marshalBody(out map[string]any, spent string) ([]byte, error) {
continue
}
dst = append(dst, '[')
+ list := reflect.ValueOf(out[k])
for j, it := range items {
if j > 0 {
dst = append(dst, ',')
}
- dst = append(dst, it...)
+ if it != nil {
+ dst = append(dst, it...)
+ continue
+ }
+ // measured only: written now, straight into the body
+ w := appendWriter{dst}
+ if err := json.NewEncoder(&w).Encode(list.Index(j).Addr().Interface()); err != nil {
+ return nil, err
+ }
+ dst = w.b[:len(w.b)-1] // Encode's newline
+ if k == spent {
+ list.Index(j).SetZero()
+ }
}
dst = append(dst, ']')
}
@@ -1518,25 +1535,57 @@ func isSplitList(v any) bool {
// marshalItems is v as json.Marshal writes it: its items each on their
// own for a list isSplitList takes, or one piece; false when json.Marshal
-// refuses any of it. spend sets each item written to its zero value.
-func marshalItems(v any, spend bool) ([]json.RawMessage, bool) {
+// refuses any of it, and the bytes they come to. An item is kept while
+// *kept, the bytes left to keep, is above 0; past it an item is only
+// measured, nil among the items, and left as it was. spend sets each item
+// kept to its zero value.
+func marshalItems(v any, spend bool, kept *int) ([]json.RawMessage, int, bool) {
if !isSplitList(v) {
b, err := json.Marshal(v)
- return []json.RawMessage{b}, err == nil
+ *kept -= len(b)
+ return []json.RawMessage{b}, len(b), err == nil
}
rv := reflect.ValueOf(v)
items := make([]json.RawMessage, rv.Len())
+ var count countWriter
+ measure := json.NewEncoder(&count)
+ size := 0
for i := range items {
// by its address, as json.Marshal reaches a slice's items: a
// MarshalJSON on the pointer is called
- b, err := json.Marshal(rv.Index(i).Addr().Interface())
+ it := rv.Index(i).Addr().Interface()
+ if *kept <= 0 {
+ before := count.n
+ if measure.Encode(it) != nil {
+ return nil, 0, false
+ }
+ size += count.n - before - 1 // Encode's newline
+ continue
+ }
+ b, err := json.Marshal(it)
if err != nil {
- return nil, false
+ return nil, 0, false
}
items[i] = b
+ size += len(b)
+ *kept -= len(b)
if spend {
rv.Index(i).SetZero()
}
}
- return items, true
+ return items, size, true
}
+
+// bodyKept is how many bytes of a body's written items marshalBody keeps
+// to put in the body; past it they are measured and written again.
+var bodyKept = 4 << 20
+
+// countWriter counts what is written to it.
+type countWriter struct{ n int }
+
+func (w *countWriter) Write(b []byte) (int, error) { w.n += len(b); return len(b), nil }
+
+// appendWriter appends what is written to it to b.
+type appendWriter struct{ b []byte }
+
+func (w *appendWriter) Write(p []byte) (int, error) { w.b = append(w.b, p...); return len(p), nil }
diff --git a/internal/gateway/marshal_body_test.go b/internal/gateway/marshal_body_test.go
index 7428028db..5444d1f7c 100644
--- a/internal/gateway/marshal_body_test.go
+++ b/internal/gateway/marshal_body_test.go
@@ -21,37 +21,48 @@ func (valMarshaler) MarshalJSON() ([]byte, error) { return []byte(`"whole"`), ni
// builder puts in a body included.
func TestMarshalBodyIsJSONMarshal(t *testing.T) {
var nilList []any
- bodies := []map[string]any{
- {},
- {"model": "m", "stream": true, "max_tokens": 10, "temperature": 0.5},
- {"messages": []any{map[string]any{"role": "user", "content": "<a&b> \u2028 \u2029 \x01 é"}, map[string]any{"role": "assistant", "content": nil}}},
- {"messages": []map[string]any{{"role": "user"}, {"role": "system", "content": []any{"x", 1}}}},
- {"one": []any{"only"}, "none": []any{}, "nil": nilList, "nilv": nil},
- {"bytes": []byte("raw bytes"), "raw": json.RawMessage(`{"a":1}`), "raws": []json.RawMessage{json.RawMessage(`1`), json.RawMessage(`{"b":2}`)}},
- {"ptrs": []ptrMarshaler{{1}, {2}}, "whole": valMarshaler{1, 2, 3}, "strs": []string{"a", "<b>"}},
- {"<k>": 1, "a\u2028": 2, "z": []int{1, 2, 3}},
+ bodies := func() []map[string]any {
+ return []map[string]any{
+ {},
+ {"model": "m", "stream": true, "max_tokens": 10, "temperature": 0.5},
+ {"messages": []any{map[string]any{"role": "user", "content": "<a&b> \u2028 \u2029 \x01 é"}, map[string]any{"role": "assistant", "content": nil}}},
+ {"messages": []map[string]any{{"role": "user"}, {"role": "system", "content": []any{"x", 1}}}},
+ {"one": []any{"only"}, "none": []any{}, "nil": nilList, "nilv": nil},
+ {"bytes": []byte("raw bytes"), "raw": json.RawMessage(`{"a":1}`), "raws": []json.RawMessage{json.RawMessage(`1`), json.RawMessage(`{"b":2}`)}},
+ {"ptrs": []ptrMarshaler{{1}, {2}}, "whole": valMarshaler{1, 2, 3}, "strs": []string{"a", "<b>"}},
+ {"<k>": 1, "a\u2028": 2, "z": []int{1, 2, 3}},
+ }
}
- for i, b := range bodies {
- want, werr := json.Marshal(b)
- got, gerr := marshalBody(b, "messages")
- if (werr == nil) != (gerr == nil) || !bytes.Equal(got, want) {
- t.Errorf("body %d:\n got %s (%v)\nwant %s (%v)", i, got, gerr, want, werr)
+ // kept whole, measured from the first item on, and from the second
+ t.Cleanup(func() { bodyKept = 4 << 20 })
+ for _, kept := range []int{4 << 20, 0, 1} {
+ bodyKept = kept
+ for i, b := range bodies() { // fresh: a spent list is zeroed
+ want, werr := json.Marshal(b)
+ got, gerr := marshalBody(b, "messages")
+ if (werr == nil) != (gerr == nil) || !bytes.Equal(got, want) {
+ t.Errorf("kept %d, body %d:\n got %s (%v)\nwant %s (%v)", kept, i, got, gerr, want, werr)
+ }
}
}
- for _, bad := range []map[string]any{
- {"messages": []any{1, math.NaN()}},
- {"x": math.Inf(1)},
- {"ch": []any{"a", make(chan int)}},
- } {
- if _, err := marshalBody(bad, ""); err == nil {
- t.Errorf("marshalBody(%v) took what json.Marshal refuses", bad)
+ for _, kept := range []int{4 << 20, 0} {
+ bodyKept = kept
+ for _, bad := range []map[string]any{
+ {"messages": []any{1, math.NaN()}},
+ {"x": math.Inf(1)},
+ {"ch": []any{"a", make(chan int)}},
+ } {
+ if _, err := marshalBody(bad, ""); err == nil {
+ t.Errorf("kept %d: marshalBody(%v) took what json.Marshal refuses", kept, bad)
+ }
}
}
}
-// A 45 MB conversation is built without the heap growing past a body
-// and its items: json.Marshal's doubling buffer took it from 45 to 316
-// MiB (#4004).
+// A 45 MB conversation is built without the heap growing past the body
+// and a few MiB: json.Marshal's doubling buffer took it from 45 to 316
+// MiB, and keeping every written item for the body was a copy more
+// (#4004).
func TestMarshalBodyAllocatesTwoBodies(t *testing.T) {
if raceOn {
t.Skip("under -race sync.Pool drops what json.Marshal's encoders keep, so each item allocates afresh")
@@ -74,8 +85,10 @@ func TestMarshalBodyAllocatesTwoBodies(t *testing.T) {
stdAlloc, n := peak(func() []byte { b, _ := json.Marshal(body); return b })
ownAlloc, _ := peak(func() []byte { b, _ := marshalBody(body, ""); return b })
t.Logf("%d MiB body: json.Marshal allocated %d MiB, marshalBody %d MiB", n>>20, stdAlloc>>20, ownAlloc>>20)
- if ownAlloc > uint64(n)*5/2 {
- t.Errorf("marshalBody allocated %d MiB for a %d MiB body, want at most 2.5x", ownAlloc>>20, n>>20)
+ // the body, the items kept for it and the encoder's buffer: no copy
+ // of every item beside the body (#4004)
+ if ownAlloc > uint64(n)*3/2 {
+ t.Errorf("marshalBody allocated %d MiB for a %d MiB body, want at most 1.5x", ownAlloc>>20, n>>20)
}
}