Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a request's body goes once it is answered, not when its stream ends (#3383)
yetone106a245502fb
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.goindex 1d78a8107..7a1e426d4 100644--- a/internal/gateway/gateway.go+++ b/internal/gateway/gateway.go@@ -4068,10 +4068,13 @@ func (s *Server) forwardOnce(ctx context.Context, p provider.Provider, to providm := gjson.GetBytes(body, "model").String()base = provider.MantleBase(base, m, !s.fits(p.ID, mantleRouteKey(m), to))}- req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+path, bytes.NewReader(body))+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+path, nil)if err != nil {return nil, err}+ // the body goes once it is answered, not when a stream ends (#3383)+ provider.SetBody(req, body)+ defer provider.LetGo(req)req.Header.Set("Content-Type", "application/json")req.Header.Set("Accept", "application/json, text/event-stream")// magpie's, or the agent's own where the provider is set to pass itdiff --git a/internal/gateway/streaming_heap_test.go b/internal/gateway/streaming_heap_test.gonew file mode 100644index 000000000..3814fbb07--- /dev/null+++ b/internal/gateway/streaming_heap_test.go@@ -0,0 +1,98 @@+package gateway++import (+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "runtime"+ "strings"+ "sync"+ "testing"++ "github.com/yetone/magpie/internal/provider"+)++// streamingHold is a ResponseWriter that passes a reply on as it comes.+type streamingHold struct {+ h http.Header+ w io.Writer+}++func (s *streamingHold) Header() http.Header { return s.h }+func (s *streamingHold) Write(b []byte) (int, error) { return s.w.Write(b) }+func (s *streamingHold) WriteHeader(int) {}+func (s *streamingHold) Flush() {}++// Requests streaming at once each hold what their retries and+// continuations read, not the body that went upstream: that was held by+// the transport until the stream ended, and eight Claude Code requests+// of 4.7 MB translated to Chat held 152 MB mid-stream, 88 MB once it+// goes when it is answered (#3383).+func TestStreamingRequestsLetTheirSentBodyGo(t *testing.T) {+ fresh(t)+ const n = 8+ got := make(chan struct{}, n)+ release := make(chan struct{})+ up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ io.Copy(io.Discard, r.Body)+ w.Header().Set("Content-Type", "text/event-stream")+ io.WriteString(w, `data: {"id":"c","object":"chat.completion.chunk","model":"m1","choices":[{"index":0,"delta":{"role":"assistant","content":"hi"}}]}`+"\n\n")+ w.(http.Flusher).Flush()+ got <- struct{}{}+ <-release+ io.WriteString(w, `data: {"id":"c","object":"chat.completion.chunk","model":"m1","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":5}}`+"\n\ndata: [DONE]\n\n")+ }))+ t.Cleanup(up.Close)+ if err := provider.Save(provider.Provider{ID: "up", Name: "Up", Key: "k", Chat: up.URL + "/v1", Models: []string{"m1"}}); err != nil {+ t.Fatal(err)+ }+ s := New()+ var sb strings.Builder+ sb.WriteString(`{"model":"up/m1","max_tokens":100,"stream":true,"messages":[`)+ turn := strings.Repeat("<div>a & b</div> some text here ", 2000)+ for i := 0; i < 80; i++ {+ if i > 0 {+ sb.WriteString(",")+ }+ role := "user"+ if i%2 == 1 {+ role = "assistant"+ }+ fmt.Fprintf(&sb, `{"role":%q,"content":%q}`, role, turn)+ }+ sb.WriteString(`]}`)+ body := sb.String()+ h := s.Handler()+ var ms runtime.MemStats+ runtime.GC()+ runtime.ReadMemStats(&ms)+ before := ms.HeapAlloc+ var wg sync.WaitGroup+ defer wg.Wait()+ once := sync.OnceFunc(func() { close(release) })+ defer once()+ for i := 0; i < n; i++ {+ wg.Add(1)+ go func() {+ defer wg.Done()+ pr, pw := io.Pipe()+ go io.Copy(io.Discard, pr)+ h.ServeHTTP(&streamingHold{h: http.Header{}, w: pw}, httptest.NewRequest("POST", "/v1/messages", strings.NewReader(body)))+ pw.Close()+ }()+ }+ for i := 0; i < n; i++ {+ <-got+ }+ runtime.GC()+ runtime.GC()+ runtime.ReadMemStats(&ms)+ held := int64(ms.HeapAlloc) - int64(before)+ once()+ // the body as it came, for another member, and as read, for a+ // continuation: about twice the body each, the sent one 4 times+ if each := held / n; each > 3*int64(len(body)) {+ t.Errorf("%d streaming requests of %d MB hold %d MB, %.1f times the body each", n, len(body)>>20, held>>20, float64(each)/float64(len(body)))+ }+}diff --git a/internal/provider/codex_zstd.go b/internal/provider/codex_zstd.goindex 1f2e7371d..92523bfb5 100644--- a/internal/provider/codex_zstd.go+++ b/internal/provider/codex_zstd.go@@ -1,8 +1,6 @@package providerimport (- "bytes"- "io""net/http""net/url""strings"@@ -77,9 +75,7 @@ func CodexCompressed(req *http.Request, body []byte) int {return len(body) // already encoded: as Codex, never twice}z := codexZstd.EncodeAll(body, make([]byte, 0, len(body)/8+64))- req.Body = io.NopCloser(bytes.NewReader(z))- req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(z)), nil }- req.ContentLength = int64(len(z))+ SetBody(req, z)req.Header.Del("Content-Length")req.Header.Set("Content-Encoding", "zstd")return len(z)diff --git a/internal/provider/copilot_auto.go b/internal/provider/copilot_auto.goindex c40cd84f5..bb76ec58a 100644--- a/internal/provider/copilot_auto.go+++ b/internal/provider/copilot_auto.go@@ -407,9 +407,7 @@ func copilotAutoSign(ctx context.Context, app copilotApp, req *http.Request, bod}m["model"], _ = json.Marshal(a.Model)nb, _ := json.Marshal(m)- req.Body = io.NopCloser(bytes.NewReader(nb))- req.ContentLength = int64(len(nb))- req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(nb)), nil }+ SetBody(req, nb)if a.Token != "" {req.Header.Set("Copilot-Session-Token", a.Token)}diff --git a/internal/provider/google.go b/internal/provider/google.goindex f0c9f684f..476d663b1 100644--- a/internal/provider/google.go+++ b/internal/provider/google.go@@ -1389,9 +1389,7 @@ func googleProvider(g googleAccount, plan string) Provider {if err != nil {return err}- req.Body = io.NopCloser(bytes.NewReader(out))- req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(out)), nil }- req.ContentLength = int64(len(out))+ SetBody(req, out)req.Header.Set("Authorization", "Bearer "+tok)req.Header.Set("User-Agent", g.app.userAgent(model))req.Header.Del("Accept")diff --git a/internal/provider/sentbody.go b/internal/provider/sentbody.gonew file mode 100644index 000000000..d95a6406a--- /dev/null+++ b/internal/provider/sentbody.go@@ -0,0 +1,75 @@+package provider++import (+ "errors"+ "io"+ "net/http"+ "sync/atomic"+)++// A request's body is held by the transport that sends it until its+// reply's body is closed: for a stream, the whole reply long. A body+// from bytes.NewReader, with the GetBody NewRequest makes for it, keeps+// every byte of the request alive that long; dozens of agents' long+// prompts streaming at once held hundreds of MB (#3383). SetBody's+// reader lets the bytes go once they are read, and its GetBody once+// LetGo says the request was answered.++// errLetGo is GetBody's error once the request was answered.+var errLetGo = errors.New("the request's body was let go once it was answered")++// heldBody is the bytes a request's GetBody reads again, until LetGo.+type heldBody struct{ b atomic.Pointer[[]byte] }++// sentBody is one read of a request's body, which drops what it has+// read once it reaches the end. Close leaves it as it is: the transport+// may close it while another of its goroutines reads it.+type sentBody struct {+ h *heldBody+ b []byte+}++func (s *sentBody) Read(p []byte) (int, error) {+ if len(s.b) == 0 {+ s.b = nil+ return 0, io.EOF+ }+ n := copy(p, s.b)+ if s.b = s.b[n:]; len(s.b) == 0 {+ s.b = nil+ }+ return n, nil+}++func (s *sentBody) Close() error { return nil }++// SetBody makes b req's body, as NewRequest would from a bytes.Reader,+// read again by GetBody for a retry or a redirect until LetGo.+func SetBody(req *http.Request, b []byte) {+ req.ContentLength = int64(len(b))+ if len(b) == 0 {+ req.Body, req.GetBody = http.NoBody, func() (io.ReadCloser, error) { return http.NoBody, nil }+ return+ }+ h := &heldBody{}+ h.b.Store(&b)+ req.Body = &sentBody{h: h, b: b}+ req.GetBody = func() (io.ReadCloser, error) {+ p := h.b.Load()+ if p == nil {+ return nil, errLetGo+ }+ return &sentBody{h: h, b: *p}, nil+ }+}++// LetGo lets go of the bytes SetBody gave req once it was answered and+// nothing will ask it again; a body SetBody didn't give is left as it is.+func LetGo(req *http.Request) {+ if req == nil {+ return+ }+ if s, ok := req.Body.(*sentBody); ok {+ s.h.b.Store(nil)+ }+}diff --git a/internal/provider/sentbody_test.go b/internal/provider/sentbody_test.gonew file mode 100644index 000000000..6ad916868--- /dev/null+++ b/internal/provider/sentbody_test.go@@ -0,0 +1,63 @@+package provider++import (+ "io"+ "net/http"+ "runtime"+ "testing"+ "weak"+)++// A request's body is held by the transport until its reply's body is+// closed, a stream's whole length (#3383): SetBody's bytes can be let go+// once they are read and LetGo says the request was answered, while the+// request is still held, and until then GetBody reads them again whole.+func TestSetBodyLetsItsBytesGo(t *testing.T) {+ b := make([]byte, 4<<20)+ for i := range b {+ b[i] = byte('a' + i%26)+ }+ alive := weak.Make(&b[0])+ req, err := http.NewRequest(http.MethodPost, "http://example.invalid/v1", nil)+ if err != nil {+ t.Fatal(err)+ }+ SetBody(req, b)+ b = nil+ if req.ContentLength != 4<<20 {+ t.Fatalf("ContentLength %d", req.ContentLength)+ }+ if got, _ := io.ReadAll(req.Body); len(got) != 4<<20 || got[27] != 'b' {+ t.Fatalf("read %d bytes", len(got))+ }+ again, err := req.GetBody()+ if err != nil {+ t.Fatal(err)+ }+ if got, _ := io.ReadAll(again); len(got) != 4<<20 {+ t.Fatalf("GetBody read %d bytes, want the whole body again", len(got))+ }+ runtime.GC()+ if alive.Value() == nil {+ t.Fatal("the body was let go before LetGo, with GetBody still to read it")+ }+ LetGo(req)+ runtime.GC()+ if alive.Value() != nil {+ t.Error("the request holds its body's bytes after it was answered")+ }+ if _, err := req.GetBody(); err == nil {+ t.Error("GetBody read a body that was let go")+ }+ runtime.KeepAlive(req)+}++// An empty body is http.NoBody, as NewRequest makes it.+func TestSetBodyEmpty(t *testing.T) {+ req, _ := http.NewRequest(http.MethodPost, "http://example.invalid/v1", nil)+ SetBody(req, nil)+ if req.Body != http.NoBody || req.ContentLength != 0 {+ t.Errorf("body %T, length %d", req.Body, req.ContentLength)+ }+ LetGo(req)+}