Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a request's body goes once it is answered, not when its stream ends (#3383)

yetone106a245502fb
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go
index 1d78a8107..7a1e426d4 100644
--- a/internal/gateway/gateway.go
+++ b/internal/gateway/gateway.go
@@ -4068,10 +4068,13 @@ func (s *Server) forwardOnce(ctx context.Context, p provider.Provider, to provid
m := gjson.GetBytes(body, "model").String()
base = provider.MantleBase(base, m, !s.fits(p.ID, mantleRouteKey(m), to))
}
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+path, bytes.NewReader(body))
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+path, nil)
if err != nil {
return nil, err
}
+ // the body goes once it is answered, not when a stream ends (#3383)
+ provider.SetBody(req, body)
+ defer provider.LetGo(req)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json, text/event-stream")
// magpie's, or the agent's own where the provider is set to pass it
diff --git a/internal/gateway/streaming_heap_test.go b/internal/gateway/streaming_heap_test.go
new file mode 100644
index 000000000..3814fbb07
--- /dev/null
+++ b/internal/gateway/streaming_heap_test.go
@@ -0,0 +1,98 @@
+package gateway
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "runtime"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/yetone/magpie/internal/provider"
+)
+
+// streamingHold is a ResponseWriter that passes a reply on as it comes.
+type streamingHold struct {
+ h http.Header
+ w io.Writer
+}
+
+func (s *streamingHold) Header() http.Header { return s.h }
+func (s *streamingHold) Write(b []byte) (int, error) { return s.w.Write(b) }
+func (s *streamingHold) WriteHeader(int) {}
+func (s *streamingHold) Flush() {}
+
+// Requests streaming at once each hold what their retries and
+// continuations read, not the body that went upstream: that was held by
+// the transport until the stream ended, and eight Claude Code requests
+// of 4.7 MB translated to Chat held 152 MB mid-stream, 88 MB once it
+// goes when it is answered (#3383).
+func TestStreamingRequestsLetTheirSentBodyGo(t *testing.T) {
+ fresh(t)
+ const n = 8
+ got := make(chan struct{}, n)
+ release := make(chan struct{})
+ up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ io.Copy(io.Discard, r.Body)
+ w.Header().Set("Content-Type", "text/event-stream")
+ io.WriteString(w, `data: {"id":"c","object":"chat.completion.chunk","model":"m1","choices":[{"index":0,"delta":{"role":"assistant","content":"hi"}}]}`+"\n\n")
+ w.(http.Flusher).Flush()
+ got <- struct{}{}
+ <-release
+ io.WriteString(w, `data: {"id":"c","object":"chat.completion.chunk","model":"m1","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":5}}`+"\n\ndata: [DONE]\n\n")
+ }))
+ t.Cleanup(up.Close)
+ if err := provider.Save(provider.Provider{ID: "up", Name: "Up", Key: "k", Chat: up.URL + "/v1", Models: []string{"m1"}}); err != nil {
+ t.Fatal(err)
+ }
+ s := New()
+ var sb strings.Builder
+ sb.WriteString(`{"model":"up/m1","max_tokens":100,"stream":true,"messages":[`)
+ turn := strings.Repeat("<div>a & b</div> some text here ", 2000)
+ for i := 0; i < 80; i++ {
+ if i > 0 {
+ sb.WriteString(",")
+ }
+ role := "user"
+ if i%2 == 1 {
+ role = "assistant"
+ }
+ fmt.Fprintf(&sb, `{"role":%q,"content":%q}`, role, turn)
+ }
+ sb.WriteString(`]}`)
+ body := sb.String()
+ h := s.Handler()
+ var ms runtime.MemStats
+ runtime.GC()
+ runtime.ReadMemStats(&ms)
+ before := ms.HeapAlloc
+ var wg sync.WaitGroup
+ defer wg.Wait()
+ once := sync.OnceFunc(func() { close(release) })
+ defer once()
+ for i := 0; i < n; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ pr, pw := io.Pipe()
+ go io.Copy(io.Discard, pr)
+ h.ServeHTTP(&streamingHold{h: http.Header{}, w: pw}, httptest.NewRequest("POST", "/v1/messages", strings.NewReader(body)))
+ pw.Close()
+ }()
+ }
+ for i := 0; i < n; i++ {
+ <-got
+ }
+ runtime.GC()
+ runtime.GC()
+ runtime.ReadMemStats(&ms)
+ held := int64(ms.HeapAlloc) - int64(before)
+ once()
+ // the body as it came, for another member, and as read, for a
+ // continuation: about twice the body each, the sent one 4 times
+ if each := held / n; each > 3*int64(len(body)) {
+ t.Errorf("%d streaming requests of %d MB hold %d MB, %.1f times the body each", n, len(body)>>20, held>>20, float64(each)/float64(len(body)))
+ }
+}
diff --git a/internal/provider/codex_zstd.go b/internal/provider/codex_zstd.go
index 1f2e7371d..92523bfb5 100644
--- a/internal/provider/codex_zstd.go
+++ b/internal/provider/codex_zstd.go
@@ -1,8 +1,6 @@
package provider
import (
- "bytes"
- "io"
"net/http"
"net/url"
"strings"
@@ -77,9 +75,7 @@ func CodexCompressed(req *http.Request, body []byte) int {
return len(body) // already encoded: as Codex, never twice
}
z := codexZstd.EncodeAll(body, make([]byte, 0, len(body)/8+64))
- req.Body = io.NopCloser(bytes.NewReader(z))
- req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(z)), nil }
- req.ContentLength = int64(len(z))
+ SetBody(req, z)
req.Header.Del("Content-Length")
req.Header.Set("Content-Encoding", "zstd")
return len(z)
diff --git a/internal/provider/copilot_auto.go b/internal/provider/copilot_auto.go
index c40cd84f5..bb76ec58a 100644
--- a/internal/provider/copilot_auto.go
+++ b/internal/provider/copilot_auto.go
@@ -407,9 +407,7 @@ func copilotAutoSign(ctx context.Context, app copilotApp, req *http.Request, bod
}
m["model"], _ = json.Marshal(a.Model)
nb, _ := json.Marshal(m)
- req.Body = io.NopCloser(bytes.NewReader(nb))
- req.ContentLength = int64(len(nb))
- req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(nb)), nil }
+ SetBody(req, nb)
if a.Token != "" {
req.Header.Set("Copilot-Session-Token", a.Token)
}
diff --git a/internal/provider/google.go b/internal/provider/google.go
index f0c9f684f..476d663b1 100644
--- a/internal/provider/google.go
+++ b/internal/provider/google.go
@@ -1389,9 +1389,7 @@ func googleProvider(g googleAccount, plan string) Provider {
if err != nil {
return err
}
- req.Body = io.NopCloser(bytes.NewReader(out))
- req.GetBody = func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(out)), nil }
- req.ContentLength = int64(len(out))
+ SetBody(req, out)
req.Header.Set("Authorization", "Bearer "+tok)
req.Header.Set("User-Agent", g.app.userAgent(model))
req.Header.Del("Accept")
diff --git a/internal/provider/sentbody.go b/internal/provider/sentbody.go
new file mode 100644
index 000000000..d95a6406a
--- /dev/null
+++ b/internal/provider/sentbody.go
@@ -0,0 +1,75 @@
+package provider
+
+import (
+ "errors"
+ "io"
+ "net/http"
+ "sync/atomic"
+)
+
+// A request's body is held by the transport that sends it until its
+// reply's body is closed: for a stream, the whole reply long. A body
+// from bytes.NewReader, with the GetBody NewRequest makes for it, keeps
+// every byte of the request alive that long; dozens of agents' long
+// prompts streaming at once held hundreds of MB (#3383). SetBody's
+// reader lets the bytes go once they are read, and its GetBody once
+// LetGo says the request was answered.
+
+// errLetGo is GetBody's error once the request was answered.
+var errLetGo = errors.New("the request's body was let go once it was answered")
+
+// heldBody is the bytes a request's GetBody reads again, until LetGo.
+type heldBody struct{ b atomic.Pointer[[]byte] }
+
+// sentBody is one read of a request's body, which drops what it has
+// read once it reaches the end. Close leaves it as it is: the transport
+// may close it while another of its goroutines reads it.
+type sentBody struct {
+ h *heldBody
+ b []byte
+}
+
+func (s *sentBody) Read(p []byte) (int, error) {
+ if len(s.b) == 0 {
+ s.b = nil
+ return 0, io.EOF
+ }
+ n := copy(p, s.b)
+ if s.b = s.b[n:]; len(s.b) == 0 {
+ s.b = nil
+ }
+ return n, nil
+}
+
+func (s *sentBody) Close() error { return nil }
+
+// SetBody makes b req's body, as NewRequest would from a bytes.Reader,
+// read again by GetBody for a retry or a redirect until LetGo.
+func SetBody(req *http.Request, b []byte) {
+ req.ContentLength = int64(len(b))
+ if len(b) == 0 {
+ req.Body, req.GetBody = http.NoBody, func() (io.ReadCloser, error) { return http.NoBody, nil }
+ return
+ }
+ h := &heldBody{}
+ h.b.Store(&b)
+ req.Body = &sentBody{h: h, b: b}
+ req.GetBody = func() (io.ReadCloser, error) {
+ p := h.b.Load()
+ if p == nil {
+ return nil, errLetGo
+ }
+ return &sentBody{h: h, b: *p}, nil
+ }
+}
+
+// LetGo lets go of the bytes SetBody gave req once it was answered and
+// nothing will ask it again; a body SetBody didn't give is left as it is.
+func LetGo(req *http.Request) {
+ if req == nil {
+ return
+ }
+ if s, ok := req.Body.(*sentBody); ok {
+ s.h.b.Store(nil)
+ }
+}
diff --git a/internal/provider/sentbody_test.go b/internal/provider/sentbody_test.go
new file mode 100644
index 000000000..6ad916868
--- /dev/null
+++ b/internal/provider/sentbody_test.go
@@ -0,0 +1,63 @@
+package provider
+
+import (
+ "io"
+ "net/http"
+ "runtime"
+ "testing"
+ "weak"
+)
+
+// A request's body is held by the transport until its reply's body is
+// closed, a stream's whole length (#3383): SetBody's bytes can be let go
+// once they are read and LetGo says the request was answered, while the
+// request is still held, and until then GetBody reads them again whole.
+func TestSetBodyLetsItsBytesGo(t *testing.T) {
+ b := make([]byte, 4<<20)
+ for i := range b {
+ b[i] = byte('a' + i%26)
+ }
+ alive := weak.Make(&b[0])
+ req, err := http.NewRequest(http.MethodPost, "http://example.invalid/v1", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ SetBody(req, b)
+ b = nil
+ if req.ContentLength != 4<<20 {
+ t.Fatalf("ContentLength %d", req.ContentLength)
+ }
+ if got, _ := io.ReadAll(req.Body); len(got) != 4<<20 || got[27] != 'b' {
+ t.Fatalf("read %d bytes", len(got))
+ }
+ again, err := req.GetBody()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got, _ := io.ReadAll(again); len(got) != 4<<20 {
+ t.Fatalf("GetBody read %d bytes, want the whole body again", len(got))
+ }
+ runtime.GC()
+ if alive.Value() == nil {
+ t.Fatal("the body was let go before LetGo, with GetBody still to read it")
+ }
+ LetGo(req)
+ runtime.GC()
+ if alive.Value() != nil {
+ t.Error("the request holds its body's bytes after it was answered")
+ }
+ if _, err := req.GetBody(); err == nil {
+ t.Error("GetBody read a body that was let go")
+ }
+ runtime.KeepAlive(req)
+}
+
+// An empty body is http.NoBody, as NewRequest makes it.
+func TestSetBodyEmpty(t *testing.T) {
+ req, _ := http.NewRequest(http.MethodPost, "http://example.invalid/v1", nil)
+ SetBody(req, nil)
+ if req.Body != http.NoBody || req.ContentLength != 0 {
+ t.Errorf("body %T, length %d", req.Body, req.ContentLength)
+ }
+ LetGo(req)
+}