Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a request's body is read in place, not copied by every reader (yg6888 on X: ~1 GB)

yetone68133754a306
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.go
index b9f4acf0a..0bc78ac97 100644
--- a/internal/gateway/fastjson.go
+++ b/internal/gateway/fastjson.go
@@ -2,6 +2,7 @@ package gateway
import (
"bytes"
+ "context"
"encoding"
"encoding/binary"
"encoding/json"
@@ -14,6 +15,7 @@ import (
"unicode"
"unicode/utf16"
"unicode/utf8"
+ "unsafe"
)
// decodeJSON is json.Unmarshal(data, dst), made for a request body: one
@@ -65,6 +67,86 @@ func decodeJSON(data []byte, dst any) error {
return nil
}
+// heldBodies are the request bodies being served (holdBody). A raw value
+// decodeJSON finds inside one of them is that body's own bytes, not a
+// copy. Each reader of a request decodes its messages into raw values —
+// the images, the conversation's id, the prompt, the edits, the chat
+// parse — and each copied every message: a 1.5 MB body allocated ~60 MB
+// on its way through a group, and three agents sending such bodies at once
+// held the heap at ~310 MB (yg6888 on X: ~1 GB). A copy is still made of
+// anything else decoded — a buffer a stream fills again, a scanner's line,
+// a body built and changed in place — since a raw value of it could be
+// written over under its holder.
+var heldBodies struct {
+ sync.Mutex
+ list atomic.Pointer[[][]byte]
+}
+
+// heldBodiesMost bounds heldBodies: a body past it is decoded with copies, as
+// before, rather than the list growing with requests whose context never
+// ends (a test's, a caller's own).
+const heldBodiesMost = 64
+
+// heldShareMost bounds how much bigger than a shared value the body it
+// keeps alive may be (inHeldBody).
+const heldShareMost = 16
+
+// holdBody shares body's bytes with what decodeJSON reads from it until
+// ctx is done. body must be the request's as read, which nothing writes
+// to: a request body is read whole once (readRequestBody) and only read
+// after. The list keeps body itself, not its address, so its memory can't
+// be handed to another allocation while it is listed.
+func holdBody(ctx context.Context, body []byte) {
+ if len(body) < validMemoMin || ctx.Done() == nil {
+ return // a short body's copies cost little; one never released would stay listed
+ }
+ heldBodies.Lock()
+ old := heldBodies.list.Load()
+ var cur [][]byte
+ if old != nil {
+ cur = *old
+ }
+ if len(cur) >= heldBodiesMost {
+ heldBodies.Unlock()
+ return
+ }
+ next := append(slices.Clip(cur), body)
+ heldBodies.list.Store(&next)
+ heldBodies.Unlock()
+ context.AfterFunc(ctx, func() {
+ heldBodies.Lock()
+ defer heldBodies.Unlock()
+ cur := *heldBodies.list.Load()
+ i := slices.IndexFunc(cur, func(b []byte) bool { return unsafe.SliceData(b) == unsafe.SliceData(body) })
+ if i < 0 {
+ return
+ }
+ next := slices.Delete(slices.Clone(cur), i, i+1)
+ heldBodies.list.Store(&next)
+ })
+}
+
+// inHeldBody says whether raw lies within a body holdBody holds and is
+// at least 1/heldShareMost of it. A value kept past its request — a
+// cache's entry, a remembered signature — keeps the whole body it shares
+// alive, so a small one is still copied: a kept value holds at most
+// heldShareMost times its own size. The values worth sharing are the big
+// ones, the messages and the long turns, which each reader copied whole.
+func inHeldBody(raw []byte) bool {
+ l := heldBodies.list.Load()
+ if l == nil || len(raw) == 0 {
+ return false
+ }
+ p := uintptr(unsafe.Pointer(unsafe.SliceData(raw)))
+ for _, b := range *l {
+ s := uintptr(unsafe.Pointer(unsafe.SliceData(b)))
+ if p >= s && p+uintptr(len(raw)) <= s+uintptr(len(b)) {
+ return len(raw)*heldShareMost >= len(b)
+ }
+ }
+ return false
+}
+
// misfit is the closing bracket of the array or object that open starts
// when a value of type t can't hold it, or 0.
func misfit(open byte, t reflect.Type) byte {
@@ -186,7 +268,13 @@ func fillJSON(raw []byte, v reflect.Value) error { return fillJSONIn(raw, v, nil
func fillJSONIn(raw []byte, v reflect.Value, members []memberSpan) error {
t := v.Type()
if t == rawMessageType {
- v.SetBytes(bytes.Clone(raw)) // a copy, as encoding/json makes
+ if inHeldBody(raw) {
+ // the request's own body, which nothing writes to: shared,
+ // and capped so an append to it can't write over the body
+ v.SetBytes(raw[:len(raw):len(raw)])
+ } else {
+ v.SetBytes(bytes.Clone(raw)) // a copy, as encoding/json makes
+ }
return nil
}
if t.Kind() != reflect.Pointer && (t.Implements(jsonUnmarshalerType) || t.Implements(textUnmarshalerType) ||
@@ -433,6 +521,23 @@ func eachElement(arr []byte, f func(el []byte) bool) {
}
}
+// elementsOf is the elements of raw, a valid JSON value, when it is an array
+// (nil otherwise), each a capped slice of raw so an append never writes
+// into it. A reader that only looks at a list's items takes them this way
+// rather than decoding a []json.RawMessage, which copies each one that
+// doesn't share a held body.
+func elementsOf(raw []byte) []json.RawMessage {
+ if len(raw) == 0 || raw[0] != '[' {
+ return nil
+ }
+ var out []json.RawMessage
+ eachElement(raw, func(el []byte) bool {
+ out = append(out, el)
+ return true
+ })
+ return out
+}
+
// skipJSONString is the end of the valid JSON string at i.
func skipJSONString(data []byte, i int) int {
i++
diff --git a/internal/gateway/fastjson_held_test.go b/internal/gateway/fastjson_held_test.go
new file mode 100644
index 000000000..2944c3386
--- /dev/null
+++ b/internal/gateway/fastjson_held_test.go
@@ -0,0 +1,135 @@
+package gateway
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+ "unsafe"
+
+ "github.com/yetone/magpie/internal/provider"
+)
+
+// sharesBody says whether part's bytes lie inside whole's.
+func sharesBody(part, whole []byte) bool {
+ p, s := uintptr(unsafe.Pointer(unsafe.SliceData(part))), uintptr(unsafe.Pointer(unsafe.SliceData(whole)))
+ return len(part) > 0 && p >= s && p+uintptr(len(part)) <= s+uintptr(len(whole))
+}
+
+// heldCount is how many bodies heldBodies lists now.
+func heldCount() int {
+ if l := heldBodies.list.Load(); l != nil {
+ return len(*l)
+ }
+ return 0
+}
+
+// A request's messages decoded as raw values, as each of its readers
+// decodes them (the images, the conversation's id, the prompt, the edits),
+// are the body's own bytes while it is served, not a copy each: a 1.5 MB
+// body allocated ~60 MB on its way through a group, and three agents
+// sending such bodies held the heap at ~310 MB. Once the request is done,
+// a decode copies again, and an append to a shared value never writes into
+// the body.
+func TestRequestBodyRawValuesShareIt(t *testing.T) {
+ before := heldCount()
+ sent := memoBody(t.Name())
+ ctx, done := context.WithCancel(context.Background())
+ r := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewReader(sent)).WithContext(ctx)
+ body, ok := (&Server{}).readRequestBody(httptest.NewRecorder(), r, provider.Chat, 0)
+ if !ok {
+ t.Fatal("body not read")
+ }
+ var q struct {
+ Messages []json.RawMessage `json:"messages"`
+ }
+ if err := decodeJSON(body, &q); err != nil || len(q.Messages) != 1 {
+ t.Fatalf("decode: %v, %d messages", err, len(q.Messages))
+ }
+ if !sharesBody(q.Messages[0], body) {
+ t.Fatal("a message decoded from the request's body while it is served is a copy of it")
+ }
+ var m map[string]json.RawMessage
+ if err := decodeJSON(q.Messages[0], &m); err != nil || !sharesBody(m["content"], body) {
+ t.Fatalf("a value decoded from a shared one is a copy (%v)", err)
+ }
+ var small struct {
+ Model json.RawMessage `json:"model"`
+ }
+ if err := decodeJSON(body, &small); err != nil || sharesBody(small.Model, body) {
+ t.Fatalf("a small value shares the body, which it would keep alive if kept (%v)", err)
+ }
+ // the body whole, shared twice: an append to one never shows in the
+ // other, through the room past the body's end
+ var w1, w2 json.RawMessage
+ if decodeJSON(body, &w1) != nil || decodeJSON(body, &w2) != nil || !sharesBody(w1, body) {
+ t.Fatal("the body decoded whole is not shared")
+ }
+ a1, a2 := append(w1, 'x'), append(w2, 'y')
+ if a1[len(body)] != 'x' || a2[len(body)] != 'y' {
+ t.Fatal("two appends to the shared body wrote over each other")
+ }
+ kept := bytes.Clone(body)
+ _ = append(m["content"], `,"x"`...)
+ if !bytes.Equal(body, kept) {
+ t.Fatal("an append to a shared value wrote into the body")
+ }
+
+ done()
+ for deadline := time.Now().Add(5 * time.Second); heldCount() > before; {
+ if time.Now().After(deadline) {
+ t.Fatal("the body is still held after its request ended")
+ }
+ time.Sleep(time.Millisecond)
+ }
+ q.Messages = nil
+ if err := decodeJSON(body, &q); err != nil || sharesBody(q.Messages[0], body) {
+ t.Fatalf("a body whose request ended is still shared (%v)", err)
+ }
+}
+
+// Only a request's own body is shared: bytes decodeJSON is handed from
+// anywhere else — a buffer a stream fills again — are copied, so a raw
+// value outlives what is written over them.
+func TestDecodeJSONCopiesWhatIsNotHeld(t *testing.T) {
+ buf := []byte(`{"messages":[{"role":"user","content":"` + strings.Repeat("x", validMemoMin) + `"}]}`)
+ var q struct {
+ Messages []json.RawMessage `json:"messages"`
+ }
+ if err := decodeJSON(buf, &q); err != nil {
+ t.Fatal(err)
+ }
+ want := string(q.Messages[0])
+ copy(buf, bytes.Repeat([]byte("y"), len(buf)))
+ if string(q.Messages[0]) != want {
+ t.Fatal("a raw value decoded from bytes not held changed with them")
+ }
+}
+
+// heldBodies is bounded: requests whose context never ends, or more than
+// heldBodiesMost at once, go without sharing rather than growing the list.
+func TestHeldBodiesBounded(t *testing.T) {
+ before := heldCount()
+ holdBody(context.Background(), memoBody(t.Name()))
+ if heldCount() != before {
+ t.Fatal("a body whose context never ends was held")
+ }
+ ctx, done := context.WithCancel(context.Background())
+ defer done()
+ for i := range heldBodiesMost + 10 {
+ holdBody(ctx, memoBody(t.Name()+string(rune('a'+i%26))))
+ }
+ if n := heldCount(); n > heldBodiesMost {
+ t.Fatalf("%d bodies held, past %d", n, heldBodiesMost)
+ }
+ done()
+ for deadline := time.Now().Add(5 * time.Second); heldCount() > before; {
+ if time.Now().After(deadline) {
+ t.Fatalf("%d bodies still held after their requests ended", heldCount()-before)
+ }
+ time.Sleep(time.Millisecond)
+ }
+}
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go
index dcd6568df..14aa0a3d8 100644
--- a/internal/gateway/gateway.go
+++ b/internal/gateway/gateway.go
@@ -6382,20 +6382,22 @@ func conversationID(in http.Header, body []byte) string {
}
}
var m struct {
- Messages []json.RawMessage `json:"messages"`
- Input json.RawMessage `json:"input"`
- Contents []json.RawMessage `json:"contents"`
+ Messages json.RawMessage `json:"messages"`
+ Input json.RawMessage `json:"input"`
+ Contents json.RawMessage `json:"contents"`
}
// An undecodable body still gets an id: the hash of the whole body.
_ = decodeJSON(body, &m)
- items := m.Messages
- geminiContents := len(items) == 0 && len(m.Contents) > 0
+ // The turns are the body's own bytes, not a copy of each: this runs on
+ // every request, and a 1.5 MB conversation's copy was ~2.8 MB of each
+ // request's allocations (yg6888 on X).
+ items, contents := elementsOf(m.Messages), elementsOf(m.Contents)
+ geminiContents := len(items) == 0 && len(contents) > 0
if geminiContents {
- items = m.Contents // Gemini generateContent repeats the first user turn.
+ items = contents // Gemini generateContent repeats the first user turn.
}
- if len(items) == 0 && len(m.Input) > 0 && m.Input[0] == '[' {
- // A malformed input array leaves items empty, and the whole body is hashed.
- _ = decodeJSON(m.Input, &items)
+ if len(items) == 0 {
+ items = elementsOf(m.Input)
}
first := []byte(m.Input)
if len(items) > 0 {
diff --git a/internal/gateway/image_input.go b/internal/gateway/image_input.go
index 991aad182..5cf69f9c2 100644
--- a/internal/gateway/image_input.go
+++ b/internal/gateway/image_input.go
@@ -78,6 +78,12 @@ func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.R
case provider.Gemini:
turnsKey, contentKey = "contents", "parts"
}
+ if raw := request[turnsKey]; len(raw) > 0 && raw[0] == '[' && !anyImageBlock(proto, raw, contentKey) {
+ // Most requests have no image, and this walk runs several times
+ // on each: decoding every turn and block cost ~2.6 MB for each
+ // 1.5 MB conversation (yg6888 on X).
+ return body, nil
+ }
var turns []json.RawMessage
if decodeJSON(request[turnsKey], &turns) != nil {
return body, nil
@@ -119,6 +125,58 @@ func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.R
return json.Marshal(request)
}
+// anyImageBlock is whether turns, a valid JSON array, may hold a block
+// walkImages gives to swap, read in place without decoding: a turn's
+// contentKey (or a Responses function_call_output's output) that is a list
+// with a block among it of an image's type, an Anthropic tool_result, or
+// a Gemini part with inlineData, fileData or a functionResponse. It errs
+// toward yes, as for a key spelled twice.
+func anyImageBlock(proto provider.Protocol, turns []byte, contentKey string) bool {
+ found := false
+ eachElement(turns, func(turn []byte) bool {
+ if turn[0] != '{' {
+ return true
+ }
+ eachMember(turn, func(key, val []byte) bool {
+ if val[0] == '[' && (jsonStringIs(key, contentKey) || (proto == provider.Responses && jsonStringIs(key, "output"))) {
+ found = anyImageIn(proto, val)
+ }
+ return !found
+ })
+ return !found
+ })
+ return found
+}
+
+// anyImageIn is whether blocks, a valid JSON array, has a block
+// swapImageBlocks would look into (anyImageBlock).
+func anyImageIn(proto provider.Protocol, blocks []byte) bool {
+ kind := "image" // Anthropic's
+ switch proto {
+ case provider.Chat:
+ kind = "image_url"
+ case provider.Responses:
+ kind = "input_image"
+ }
+ found := false
+ eachElement(blocks, func(block []byte) bool {
+ if block[0] != '{' {
+ return true
+ }
+ eachMember(block, func(key, val []byte) bool {
+ switch {
+ case proto == provider.Gemini:
+ found = jsonStringIs(key, "inlineData") || jsonStringIs(key, "fileData") || jsonStringIs(key, "functionResponse")
+ case jsonStringIs(key, "type"):
+ found = jsonStringIs(val, kind) || (proto == provider.Anthropic && jsonStringIs(val, "tool_result"))
+ }
+ return !found
+ })
+ return !found
+ })
+ return found
+}
+
// swapImageBlocks swaps the images among a turn's blocks, and those in an
// Anthropic tool_result's, which are a tool's.
func swapImageBlocks(proto provider.Protocol, raw json.RawMessage, last, tool bool, swap func(imageAt) (json.RawMessage, error)) (json.RawMessage, bool, error) {
diff --git a/internal/gateway/image_input_test.go b/internal/gateway/image_input_test.go
index 89585a312..b02475dac 100644
--- a/internal/gateway/image_input_test.go
+++ b/internal/gateway/image_input_test.go
@@ -1,12 +1,14 @@
package gateway
import (
+ "context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
+ "runtime"
"strings"
"testing"
@@ -528,3 +530,58 @@ func TestTextOnlyRefusalSaysWhyAndWhere(t *testing.T) {
}
}
}
+
+// Every request's images are looked for several times (hasImage,
+// textOnlyBody, the vision checks), and its conversation's id hashed:
+// without an image, the turns are read where they are, not decoded turn by
+// turn and block by block. A 1.5 MB conversation cost ~2.6 MB a walk and
+// ~2.8 MB an id (yg6888 on X). An image is still found where it hides:
+// last, in a tool's output, under an escaped key, under a key given twice.
+func TestImageWalkReadsTextOnlyTurnsInPlace(t *testing.T) {
+ var turns []string
+ for i := range 200 {
+ turns = append(turns, `{"role":"user","content":[{"type":"text","text":"`+strings.Repeat("read this file ", 300)+`"}]}`,
+ `{"role":"assistant","content":"ok `+strings.Repeat("x", i)+`"}`)
+ }
+ body := []byte(`{"model":"m","messages":[` + strings.Join(turns, ",") + `]}`)
+ ctx, done := context.WithCancel(context.Background())
+ defer done()
+ holdBody(ctx, body) // as readRequestBody holds a request's
+ id := conversationID(http.Header{}, body)
+ if hasImage(provider.Chat, body) {
+ t.Fatal("an image found in a text-only body")
+ }
+ var before, after runtime.MemStats
+ runtime.ReadMemStats(&before)
+ const n = 5
+ for range n {
+ hasImage(provider.Chat, body)
+ textOnlyBody(provider.Chat, body)
+ if conversationID(http.Header{}, body) != id {
+ t.Fatal("the conversation's id changed")
+ }
+ }
+ runtime.ReadMemStats(&after)
+ if per := (after.TotalAlloc - before.TotalAlloc) / n; per > uint64(len(body))/8 {
+ t.Fatalf("two image walks and an id of a %d-byte text-only body allocated %d bytes", len(body), per)
+ }
+
+ head := `{"model":"m","messages":[` + strings.Join(turns[:4], ",") + ","
+ for i, c := range []struct {
+ proto provider.Protocol
+ body string
+ }{
+ {provider.Chat, head + `{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://x/a.png"}}]}]}`},
+ {provider.Chat, head + `{"role":"user","content":[{"%u0074ype":"%u0069mage_url","image_url":{"url":"https://x/a.png"}}]}]}`},
+ {provider.Chat, head + `{"role":"user","content":"hi","content":[{"type":"text","type":"image_url","image_url":"https://x/a.png"}]}]}`},
+ {provider.Chat, head + `{"role":"tool","tool_call_id":"c","content":[{"type":"image_url","image_url":{"url":"https://x/a.png"}}]},{"role":"user","content":"go"}]}`},
+ {provider.Responses, `{"input":[{"type":"function_call_output","call_id":"c","output":[{"type":"input_image","image_url":"https://x/a.png"}]},{"role":"user","content":"go"}]}`},
+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"t","content":[{"type":"image","source":{"type":"url","url":"https://x/a.png"}}]}]}]}`},
+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"functionResponse":{"name":"f","parts":[{"inlineData":{"mimeType":"image/png","data":"AAAA"}}]}}]}]}`},
+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"inlineData":{"mimeType":"image/png","data":"AAAA"}}]}]}`},
+ } {
+ if !hasImage(c.proto, []byte(strings.ReplaceAll(c.body, "%u", "\x5cu"))) {
+ t.Errorf("case %d (%v): its image was not found", i, c.proto)
+ }
+ }
+}
diff --git a/internal/gateway/request_bounds.go b/internal/gateway/request_bounds.go
index 51e2fd2f1..92a48048f 100644
--- a/internal/gateway/request_bounds.go
+++ b/internal/gateway/request_bounds.go
@@ -48,6 +48,7 @@ func (s *Server) readRequestBody(w http.ResponseWriter, r *http.Request, from pr
writeError(w, from, status, err.Error())
return nil, false
}
+ holdBody(r.Context(), body)
return body, true
}