Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a request's body is read in place, not copied by every reader (yg6888 on X: ~1 GB)
yetone68133754a306
diff --git a/internal/gateway/fastjson.go b/internal/gateway/fastjson.goindex b9f4acf0a..0bc78ac97 100644--- a/internal/gateway/fastjson.go+++ b/internal/gateway/fastjson.go@@ -2,6 +2,7 @@ package gatewayimport ("bytes"+ "context""encoding""encoding/binary""encoding/json"@@ -14,6 +15,7 @@ import ("unicode""unicode/utf16""unicode/utf8"+ "unsafe")// decodeJSON is json.Unmarshal(data, dst), made for a request body: one@@ -65,6 +67,86 @@ func decodeJSON(data []byte, dst any) error {return nil}+// heldBodies are the request bodies being served (holdBody). A raw value+// decodeJSON finds inside one of them is that body's own bytes, not a+// copy. Each reader of a request decodes its messages into raw values —+// the images, the conversation's id, the prompt, the edits, the chat+// parse — and each copied every message: a 1.5 MB body allocated ~60 MB+// on its way through a group, and three agents sending such bodies at once+// held the heap at ~310 MB (yg6888 on X: ~1 GB). A copy is still made of+// anything else decoded — a buffer a stream fills again, a scanner's line,+// a body built and changed in place — since a raw value of it could be+// written over under its holder.+var heldBodies struct {+ sync.Mutex+ list atomic.Pointer[[][]byte]+}++// heldBodiesMost bounds heldBodies: a body past it is decoded with copies, as+// before, rather than the list growing with requests whose context never+// ends (a test's, a caller's own).+const heldBodiesMost = 64++// heldShareMost bounds how much bigger than a shared value the body it+// keeps alive may be (inHeldBody).+const heldShareMost = 16++// holdBody shares body's bytes with what decodeJSON reads from it until+// ctx is done. body must be the request's as read, which nothing writes+// to: a request body is read whole once (readRequestBody) and only read+// after. The list keeps body itself, not its address, so its memory can't+// be handed to another allocation while it is listed.+func holdBody(ctx context.Context, body []byte) {+ if len(body) < validMemoMin || ctx.Done() == nil {+ return // a short body's copies cost little; one never released would stay listed+ }+ heldBodies.Lock()+ old := heldBodies.list.Load()+ var cur [][]byte+ if old != nil {+ cur = *old+ }+ if len(cur) >= heldBodiesMost {+ heldBodies.Unlock()+ return+ }+ next := append(slices.Clip(cur), body)+ heldBodies.list.Store(&next)+ heldBodies.Unlock()+ context.AfterFunc(ctx, func() {+ heldBodies.Lock()+ defer heldBodies.Unlock()+ cur := *heldBodies.list.Load()+ i := slices.IndexFunc(cur, func(b []byte) bool { return unsafe.SliceData(b) == unsafe.SliceData(body) })+ if i < 0 {+ return+ }+ next := slices.Delete(slices.Clone(cur), i, i+1)+ heldBodies.list.Store(&next)+ })+}++// inHeldBody says whether raw lies within a body holdBody holds and is+// at least 1/heldShareMost of it. A value kept past its request — a+// cache's entry, a remembered signature — keeps the whole body it shares+// alive, so a small one is still copied: a kept value holds at most+// heldShareMost times its own size. The values worth sharing are the big+// ones, the messages and the long turns, which each reader copied whole.+func inHeldBody(raw []byte) bool {+ l := heldBodies.list.Load()+ if l == nil || len(raw) == 0 {+ return false+ }+ p := uintptr(unsafe.Pointer(unsafe.SliceData(raw)))+ for _, b := range *l {+ s := uintptr(unsafe.Pointer(unsafe.SliceData(b)))+ if p >= s && p+uintptr(len(raw)) <= s+uintptr(len(b)) {+ return len(raw)*heldShareMost >= len(b)+ }+ }+ return false+}+// misfit is the closing bracket of the array or object that open starts// when a value of type t can't hold it, or 0.func misfit(open byte, t reflect.Type) byte {@@ -186,7 +268,13 @@ func fillJSON(raw []byte, v reflect.Value) error { return fillJSONIn(raw, v, nilfunc fillJSONIn(raw []byte, v reflect.Value, members []memberSpan) error {t := v.Type()if t == rawMessageType {- v.SetBytes(bytes.Clone(raw)) // a copy, as encoding/json makes+ if inHeldBody(raw) {+ // the request's own body, which nothing writes to: shared,+ // and capped so an append to it can't write over the body+ v.SetBytes(raw[:len(raw):len(raw)])+ } else {+ v.SetBytes(bytes.Clone(raw)) // a copy, as encoding/json makes+ }return nil}if t.Kind() != reflect.Pointer && (t.Implements(jsonUnmarshalerType) || t.Implements(textUnmarshalerType) ||@@ -433,6 +521,23 @@ func eachElement(arr []byte, f func(el []byte) bool) {}}+// elementsOf is the elements of raw, a valid JSON value, when it is an array+// (nil otherwise), each a capped slice of raw so an append never writes+// into it. A reader that only looks at a list's items takes them this way+// rather than decoding a []json.RawMessage, which copies each one that+// doesn't share a held body.+func elementsOf(raw []byte) []json.RawMessage {+ if len(raw) == 0 || raw[0] != '[' {+ return nil+ }+ var out []json.RawMessage+ eachElement(raw, func(el []byte) bool {+ out = append(out, el)+ return true+ })+ return out+}+// skipJSONString is the end of the valid JSON string at i.func skipJSONString(data []byte, i int) int {i++diff --git a/internal/gateway/fastjson_held_test.go b/internal/gateway/fastjson_held_test.gonew file mode 100644index 000000000..2944c3386--- /dev/null+++ b/internal/gateway/fastjson_held_test.go@@ -0,0 +1,135 @@+package gateway++import (+ "bytes"+ "context"+ "encoding/json"+ "net/http/httptest"+ "strings"+ "testing"+ "time"+ "unsafe"++ "github.com/yetone/magpie/internal/provider"+)++// sharesBody says whether part's bytes lie inside whole's.+func sharesBody(part, whole []byte) bool {+ p, s := uintptr(unsafe.Pointer(unsafe.SliceData(part))), uintptr(unsafe.Pointer(unsafe.SliceData(whole)))+ return len(part) > 0 && p >= s && p+uintptr(len(part)) <= s+uintptr(len(whole))+}++// heldCount is how many bodies heldBodies lists now.+func heldCount() int {+ if l := heldBodies.list.Load(); l != nil {+ return len(*l)+ }+ return 0+}++// A request's messages decoded as raw values, as each of its readers+// decodes them (the images, the conversation's id, the prompt, the edits),+// are the body's own bytes while it is served, not a copy each: a 1.5 MB+// body allocated ~60 MB on its way through a group, and three agents+// sending such bodies held the heap at ~310 MB. Once the request is done,+// a decode copies again, and an append to a shared value never writes into+// the body.+func TestRequestBodyRawValuesShareIt(t *testing.T) {+ before := heldCount()+ sent := memoBody(t.Name())+ ctx, done := context.WithCancel(context.Background())+ r := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewReader(sent)).WithContext(ctx)+ body, ok := (&Server{}).readRequestBody(httptest.NewRecorder(), r, provider.Chat, 0)+ if !ok {+ t.Fatal("body not read")+ }+ var q struct {+ Messages []json.RawMessage `json:"messages"`+ }+ if err := decodeJSON(body, &q); err != nil || len(q.Messages) != 1 {+ t.Fatalf("decode: %v, %d messages", err, len(q.Messages))+ }+ if !sharesBody(q.Messages[0], body) {+ t.Fatal("a message decoded from the request's body while it is served is a copy of it")+ }+ var m map[string]json.RawMessage+ if err := decodeJSON(q.Messages[0], &m); err != nil || !sharesBody(m["content"], body) {+ t.Fatalf("a value decoded from a shared one is a copy (%v)", err)+ }+ var small struct {+ Model json.RawMessage `json:"model"`+ }+ if err := decodeJSON(body, &small); err != nil || sharesBody(small.Model, body) {+ t.Fatalf("a small value shares the body, which it would keep alive if kept (%v)", err)+ }+ // the body whole, shared twice: an append to one never shows in the+ // other, through the room past the body's end+ var w1, w2 json.RawMessage+ if decodeJSON(body, &w1) != nil || decodeJSON(body, &w2) != nil || !sharesBody(w1, body) {+ t.Fatal("the body decoded whole is not shared")+ }+ a1, a2 := append(w1, 'x'), append(w2, 'y')+ if a1[len(body)] != 'x' || a2[len(body)] != 'y' {+ t.Fatal("two appends to the shared body wrote over each other")+ }+ kept := bytes.Clone(body)+ _ = append(m["content"], `,"x"`...)+ if !bytes.Equal(body, kept) {+ t.Fatal("an append to a shared value wrote into the body")+ }++ done()+ for deadline := time.Now().Add(5 * time.Second); heldCount() > before; {+ if time.Now().After(deadline) {+ t.Fatal("the body is still held after its request ended")+ }+ time.Sleep(time.Millisecond)+ }+ q.Messages = nil+ if err := decodeJSON(body, &q); err != nil || sharesBody(q.Messages[0], body) {+ t.Fatalf("a body whose request ended is still shared (%v)", err)+ }+}++// Only a request's own body is shared: bytes decodeJSON is handed from+// anywhere else — a buffer a stream fills again — are copied, so a raw+// value outlives what is written over them.+func TestDecodeJSONCopiesWhatIsNotHeld(t *testing.T) {+ buf := []byte(`{"messages":[{"role":"user","content":"` + strings.Repeat("x", validMemoMin) + `"}]}`)+ var q struct {+ Messages []json.RawMessage `json:"messages"`+ }+ if err := decodeJSON(buf, &q); err != nil {+ t.Fatal(err)+ }+ want := string(q.Messages[0])+ copy(buf, bytes.Repeat([]byte("y"), len(buf)))+ if string(q.Messages[0]) != want {+ t.Fatal("a raw value decoded from bytes not held changed with them")+ }+}++// heldBodies is bounded: requests whose context never ends, or more than+// heldBodiesMost at once, go without sharing rather than growing the list.+func TestHeldBodiesBounded(t *testing.T) {+ before := heldCount()+ holdBody(context.Background(), memoBody(t.Name()))+ if heldCount() != before {+ t.Fatal("a body whose context never ends was held")+ }+ ctx, done := context.WithCancel(context.Background())+ defer done()+ for i := range heldBodiesMost + 10 {+ holdBody(ctx, memoBody(t.Name()+string(rune('a'+i%26))))+ }+ if n := heldCount(); n > heldBodiesMost {+ t.Fatalf("%d bodies held, past %d", n, heldBodiesMost)+ }+ done()+ for deadline := time.Now().Add(5 * time.Second); heldCount() > before; {+ if time.Now().After(deadline) {+ t.Fatalf("%d bodies still held after their requests ended", heldCount()-before)+ }+ time.Sleep(time.Millisecond)+ }+}diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.goindex dcd6568df..14aa0a3d8 100644--- a/internal/gateway/gateway.go+++ b/internal/gateway/gateway.go@@ -6382,20 +6382,22 @@ func conversationID(in http.Header, body []byte) string {}}var m struct {- Messages []json.RawMessage `json:"messages"`- Input json.RawMessage `json:"input"`- Contents []json.RawMessage `json:"contents"`+ Messages json.RawMessage `json:"messages"`+ Input json.RawMessage `json:"input"`+ Contents json.RawMessage `json:"contents"`}// An undecodable body still gets an id: the hash of the whole body._ = decodeJSON(body, &m)- items := m.Messages- geminiContents := len(items) == 0 && len(m.Contents) > 0+ // The turns are the body's own bytes, not a copy of each: this runs on+ // every request, and a 1.5 MB conversation's copy was ~2.8 MB of each+ // request's allocations (yg6888 on X).+ items, contents := elementsOf(m.Messages), elementsOf(m.Contents)+ geminiContents := len(items) == 0 && len(contents) > 0if geminiContents {- items = m.Contents // Gemini generateContent repeats the first user turn.+ items = contents // Gemini generateContent repeats the first user turn.}- if len(items) == 0 && len(m.Input) > 0 && m.Input[0] == '[' {- // A malformed input array leaves items empty, and the whole body is hashed.- _ = decodeJSON(m.Input, &items)+ if len(items) == 0 {+ items = elementsOf(m.Input)}first := []byte(m.Input)if len(items) > 0 {diff --git a/internal/gateway/image_input.go b/internal/gateway/image_input.goindex 991aad182..5cf69f9c2 100644--- a/internal/gateway/image_input.go+++ b/internal/gateway/image_input.go@@ -78,6 +78,12 @@ func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.Rcase provider.Gemini:turnsKey, contentKey = "contents", "parts"}+ if raw := request[turnsKey]; len(raw) > 0 && raw[0] == '[' && !anyImageBlock(proto, raw, contentKey) {+ // Most requests have no image, and this walk runs several times+ // on each: decoding every turn and block cost ~2.6 MB for each+ // 1.5 MB conversation (yg6888 on X).+ return body, nil+ }var turns []json.RawMessageif decodeJSON(request[turnsKey], &turns) != nil {return body, nil@@ -119,6 +125,58 @@ func walkImages(proto provider.Protocol, body []byte, swap func(imageAt) (json.Rreturn json.Marshal(request)}+// anyImageBlock is whether turns, a valid JSON array, may hold a block+// walkImages gives to swap, read in place without decoding: a turn's+// contentKey (or a Responses function_call_output's output) that is a list+// with a block among it of an image's type, an Anthropic tool_result, or+// a Gemini part with inlineData, fileData or a functionResponse. It errs+// toward yes, as for a key spelled twice.+func anyImageBlock(proto provider.Protocol, turns []byte, contentKey string) bool {+ found := false+ eachElement(turns, func(turn []byte) bool {+ if turn[0] != '{' {+ return true+ }+ eachMember(turn, func(key, val []byte) bool {+ if val[0] == '[' && (jsonStringIs(key, contentKey) || (proto == provider.Responses && jsonStringIs(key, "output"))) {+ found = anyImageIn(proto, val)+ }+ return !found+ })+ return !found+ })+ return found+}++// anyImageIn is whether blocks, a valid JSON array, has a block+// swapImageBlocks would look into (anyImageBlock).+func anyImageIn(proto provider.Protocol, blocks []byte) bool {+ kind := "image" // Anthropic's+ switch proto {+ case provider.Chat:+ kind = "image_url"+ case provider.Responses:+ kind = "input_image"+ }+ found := false+ eachElement(blocks, func(block []byte) bool {+ if block[0] != '{' {+ return true+ }+ eachMember(block, func(key, val []byte) bool {+ switch {+ case proto == provider.Gemini:+ found = jsonStringIs(key, "inlineData") || jsonStringIs(key, "fileData") || jsonStringIs(key, "functionResponse")+ case jsonStringIs(key, "type"):+ found = jsonStringIs(val, kind) || (proto == provider.Anthropic && jsonStringIs(val, "tool_result"))+ }+ return !found+ })+ return !found+ })+ return found+}+// swapImageBlocks swaps the images among a turn's blocks, and those in an// Anthropic tool_result's, which are a tool's.func swapImageBlocks(proto provider.Protocol, raw json.RawMessage, last, tool bool, swap func(imageAt) (json.RawMessage, error)) (json.RawMessage, bool, error) {diff --git a/internal/gateway/image_input_test.go b/internal/gateway/image_input_test.goindex 89585a312..b02475dac 100644--- a/internal/gateway/image_input_test.go+++ b/internal/gateway/image_input_test.go@@ -1,12 +1,14 @@package gatewayimport (+ "context""encoding/json""io""net/http""net/http/httptest""os""path/filepath"+ "runtime""strings""testing"@@ -528,3 +530,58 @@ func TestTextOnlyRefusalSaysWhyAndWhere(t *testing.T) {}}}++// Every request's images are looked for several times (hasImage,+// textOnlyBody, the vision checks), and its conversation's id hashed:+// without an image, the turns are read where they are, not decoded turn by+// turn and block by block. A 1.5 MB conversation cost ~2.6 MB a walk and+// ~2.8 MB an id (yg6888 on X). An image is still found where it hides:+// last, in a tool's output, under an escaped key, under a key given twice.+func TestImageWalkReadsTextOnlyTurnsInPlace(t *testing.T) {+ var turns []string+ for i := range 200 {+ turns = append(turns, `{"role":"user","content":[{"type":"text","text":"`+strings.Repeat("read this file ", 300)+`"}]}`,+ `{"role":"assistant","content":"ok `+strings.Repeat("x", i)+`"}`)+ }+ body := []byte(`{"model":"m","messages":[` + strings.Join(turns, ",") + `]}`)+ ctx, done := context.WithCancel(context.Background())+ defer done()+ holdBody(ctx, body) // as readRequestBody holds a request's+ id := conversationID(http.Header{}, body)+ if hasImage(provider.Chat, body) {+ t.Fatal("an image found in a text-only body")+ }+ var before, after runtime.MemStats+ runtime.ReadMemStats(&before)+ const n = 5+ for range n {+ hasImage(provider.Chat, body)+ textOnlyBody(provider.Chat, body)+ if conversationID(http.Header{}, body) != id {+ t.Fatal("the conversation's id changed")+ }+ }+ runtime.ReadMemStats(&after)+ if per := (after.TotalAlloc - before.TotalAlloc) / n; per > uint64(len(body))/8 {+ t.Fatalf("two image walks and an id of a %d-byte text-only body allocated %d bytes", len(body), per)+ }++ head := `{"model":"m","messages":[` + strings.Join(turns[:4], ",") + ","+ for i, c := range []struct {+ proto provider.Protocol+ body string+ }{+ {provider.Chat, head + `{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://x/a.png"}}]}]}`},+ {provider.Chat, head + `{"role":"user","content":[{"%u0074ype":"%u0069mage_url","image_url":{"url":"https://x/a.png"}}]}]}`},+ {provider.Chat, head + `{"role":"user","content":"hi","content":[{"type":"text","type":"image_url","image_url":"https://x/a.png"}]}]}`},+ {provider.Chat, head + `{"role":"tool","tool_call_id":"c","content":[{"type":"image_url","image_url":{"url":"https://x/a.png"}}]},{"role":"user","content":"go"}]}`},+ {provider.Responses, `{"input":[{"type":"function_call_output","call_id":"c","output":[{"type":"input_image","image_url":"https://x/a.png"}]},{"role":"user","content":"go"}]}`},+ {provider.Anthropic, `{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"t","content":[{"type":"image","source":{"type":"url","url":"https://x/a.png"}}]}]}]}`},+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"functionResponse":{"name":"f","parts":[{"inlineData":{"mimeType":"image/png","data":"AAAA"}}]}}]}]}`},+ {provider.Gemini, `{"contents":[{"role":"user","parts":[{"inlineData":{"mimeType":"image/png","data":"AAAA"}}]}]}`},+ } {+ if !hasImage(c.proto, []byte(strings.ReplaceAll(c.body, "%u", "\x5cu"))) {+ t.Errorf("case %d (%v): its image was not found", i, c.proto)+ }+ }+}diff --git a/internal/gateway/request_bounds.go b/internal/gateway/request_bounds.goindex 51e2fd2f1..92a48048f 100644--- a/internal/gateway/request_bounds.go+++ b/internal/gateway/request_bounds.go@@ -48,6 +48,7 @@ func (s *Server) readRequestBody(w http.ResponseWriter, r *http.Request, from prwriteError(w, from, status, err.Error())return nil, false}+ holdBody(r.Context(), body)return body, true}