Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

plugin: a fetch past the host's 128 streams waits for one, not a local 502 (#3890)

yetonef8cd964aec76
diff --git a/internal/gateway/plugin_stream_admit_test.go b/internal/gateway/plugin_stream_admit_test.go
new file mode 100644
index 000000000..eeb730f15
--- /dev/null
+++ b/internal/gateway/plugin_stream_admit_test.go
@@ -0,0 +1,103 @@
+package gateway
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// #3890: 192 streams at once through one plugin. The plugin host holds 128
+// streams' windows; the rest used to be refused locally at once ("too many
+// plugin calls are in flight"), each refusal a 502 try the gateway sent
+// again. Now they wait for a window: every request is answered on its
+// first try, and the vendor sees each once.
+func TestPluginStreamsPastTheBudgetWait(t *testing.T) {
+ const n = 192
+ var hits, live, most atomic.Int64
+ release := make(chan struct{})
+ pid := besideFake(t, "stream-admit", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ io.Copy(io.Discard, r.Body)
+ hits.Add(1)
+ now := live.Add(1)
+ defer live.Add(-1)
+ for m := most.Load(); now > m && !most.CompareAndSwap(m, now); m = most.Load() {
+ }
+ w.Header().Set("Content-Type", "text/event-stream")
+ w.WriteHeader(200)
+ w.(http.Flusher).Flush()
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ return
+ }
+ io.WriteString(w, `data: {"id":"a","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}`+"\n\n")
+ io.WriteString(w, `data: {"id":"a","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}`+"\n\n")
+ io.WriteString(w, "data: [DONE]\n\n")
+ }))
+ s := New()
+ gw := httptest.NewServer(s.Handler())
+ t.Cleanup(gw.Close)
+ tr := http.DefaultTransport.(*http.Transport).Clone()
+ tr.Proxy, tr.MaxIdleConnsPerHost = nil, n
+ t.Cleanup(tr.CloseIdleConnections)
+ client := &http.Client{Transport: tr, Timeout: 2 * time.Minute}
+
+ model := pid + "/fake-1"
+ var wg sync.WaitGroup
+ errs := make(chan string, n)
+ for range n {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ res, err := client.Post(gw.URL+"/v1/chat/completions", "application/json", strings.NewReader(pluginReplyRequest(model, true)))
+ if err != nil {
+ errs <- err.Error()
+ return
+ }
+ b, _ := io.ReadAll(res.Body)
+ res.Body.Close()
+ if res.StatusCode != 200 || !strings.Contains(string(b), "ok") {
+ errs <- res.Status + " " + string(b)
+ }
+ }()
+ }
+ // the host's windows are all held by streams the vendor keeps open
+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+ defer cancel()
+ for live.Load() < 128 && ctx.Err() == nil {
+ time.Sleep(10 * time.Millisecond)
+ }
+ if live.Load() < 128 {
+ t.Fatalf("the vendor holds %d streams, want the host's 128", live.Load())
+ }
+ time.Sleep(300 * time.Millisecond) // the other 64 reach the host meanwhile
+ close(release)
+ wg.Wait()
+ close(errs)
+ for e := range errs {
+ t.Error(e)
+ }
+ if h := hits.Load(); h != n {
+ t.Errorf("the vendor saw %d requests, want %d", h, n)
+ }
+ if m := most.Load(); m > 128 {
+ t.Errorf("the vendor held %d streams at once, past the host's 128", m)
+ }
+ routes := routesOf(s, model)
+ if len(routes) == 0 {
+ t.Fatal("no routes traced")
+ }
+ for _, r := range routes {
+ if len(r.Tries) != 1 || r.Tries[0].Status != 200 {
+ for _, tr := range r.Tries {
+ t.Errorf("route %d: a try %d %s", r.ID, tr.Status, tr.Error)
+ }
+ }
+ }
+}
diff --git a/internal/plugin/host.go b/internal/plugin/host.go
index 6cf8b05f1..5b311a39e 100644
--- a/internal/plugin/host.go
+++ b/internal/plugin/host.go
@@ -153,6 +153,13 @@ var (
// under it, and so is any RPC result, so a child that never ends a line
// fails the host rather than making it allocate without bound.
maxWireLine = 32 << 20
+ // streamWait is how long a fetch waits for a stream's window to come
+ // free when the budget's are all held, while its request lives. Past
+ // the budget a fetch used to fail at once (errTooManyCalls), and the
+ // gateway counted that as a 502 and tried again: 192 streams at once
+ // through one plugin gave 298 local refusals that never reached the
+ // vendor (#3890).
+ streamWait = time.Minute
// maxHeadBytes and maxHeadEntries are magpie's own head-envelope policy,
// not HTTP's: a fetch's head carries at most maxHeadBytes of header bytes
// (the UTF-8 bytes of every name and value, plus headEntryCost an entry)
@@ -1445,14 +1452,7 @@ func (h *host) dropCtrl() {
// when the host already holds its limit of live and retiring calls, so a new
// call fails clearly rather than a legitimate accepted reply being cut short.
func (h *host) begin(stream bool) (int64, *call, error) {
- h.mu.Lock()
- defer h.mu.Unlock()
- if !h.alive() {
- return 0, nil, errHostGone
- }
- c := &call{done: make(chan message, 1), stream: stream, wake: make(chan struct{}, 1), state: callSending}
if stream {
- c.window = streamWindow
// a stream holds one window of the budget as well as a call slot, so
// the RPC reserve is never spent on one more stream
select {
@@ -1461,6 +1461,46 @@ func (h *host) begin(stream bool) (int64, *call, error) {
return 0, nil, errTooManyCalls
}
}
+ return h.admit(stream)
+}
+
+// beginStream is begin for a fetch: while the budget's windows are all held
+// it waits for one to come free, for streamWait at most and only while ctx
+// lives, rather than failing a request that would have run a moment later.
+// The wait is outside h.mu, which the call that frees a window takes.
+func (h *host) beginStream(ctx context.Context) (int64, *call, error) {
+ select {
+ case h.streams <- struct{}{}:
+ default:
+ t := time.NewTimer(streamWait)
+ defer t.Stop()
+ select {
+ case h.streams <- struct{}{}:
+ case <-ctx.Done():
+ return 0, nil, context.Cause(ctx)
+ case <-h.dead:
+ return 0, nil, errHostGone
+ case <-t.C:
+ return 0, nil, errTooManyCalls
+ }
+ }
+ return h.admit(true)
+}
+
+// admit registers a call whose stream window, if it is a stream, is taken.
+func (h *host) admit(stream bool) (int64, *call, error) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if !h.alive() {
+ if stream {
+ <-h.streams
+ }
+ return 0, nil, errHostGone
+ }
+ c := &call{done: make(chan message, 1), stream: stream, wake: make(chan struct{}, 1), state: callSending}
+ if stream {
+ c.window = streamWindow
+ }
select {
case h.slots <- struct{}{}:
default:
@@ -1693,7 +1733,7 @@ func Fetch(ctx context.Context, r FetchRequest) (*http.Response, error) {
if r.Proxy == "" {
r.Proxy = proxyOf(ctx)
}
- id, c, err := h.begin(true)
+ id, c, err := h.beginStream(ctx)
if err != nil {
return nil, err
}
diff --git a/internal/plugin/host_test.go b/internal/plugin/host_test.go
index 51daeca51..bb2669e3f 100644
--- a/internal/plugin/host_test.go
+++ b/internal/plugin/host_test.go
@@ -675,6 +675,62 @@ func TestHostAdmissionLimitFailsTheNewCall(t *testing.T) {
}
}
+// TestHostFetchWaitsForAStreamWindow (#3890): a fetch past the budget waits
+// for a window to come free and then runs, rather than failing at once into
+// a local 502 the gateway tries again; it gives up when its request ends or
+// after streamWait.
+func TestHostFetchWaitsForAStreamWindow(t *testing.T) {
+ oldWindow, oldBudget := streamWindow, streamBudget
+ oldReserve, oldWait := rpcReserve, streamWait
+ streamWindow, streamBudget, rpcReserve = 1<<20, 1<<20, 0 // one streaming call
+ t.Cleanup(func() { streamWindow, streamBudget, rpcReserve, streamWait = oldWindow, oldBudget, oldReserve, oldWait })
+
+ f := newFakeHost(t)
+ first, _, err := f.host.begin(true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ type got struct {
+ id int64
+ err error
+ }
+ waited := make(chan got, 1)
+ go func() {
+ id, _, err := f.host.beginStream(context.Background())
+ waited <- got{id, err}
+ }()
+ select {
+ case g := <-waited:
+ t.Fatalf("a fetch past the budget = %d, %v, want it waiting", g.id, g.err)
+ case <-time.After(50 * time.Millisecond):
+ }
+ f.host.mu.Lock()
+ c := f.host.calls[first]
+ f.host.mu.Unlock()
+ c.release()
+ if g := <-waited; g.err != nil {
+ t.Fatalf("the waiting fetch, once a window came free: %v", g.err)
+ }
+
+ // the budget is held again: a request that ends gives the wait up
+ ctx, cancel := context.WithCancel(context.Background())
+ go func() {
+ _, _, err := f.host.beginStream(ctx)
+ waited <- got{0, err}
+ }()
+ cancel()
+ if g := <-waited; !errors.Is(g.err, context.Canceled) {
+ t.Fatalf("a fetch whose request ended = %v, want canceled", g.err)
+ }
+ streamWait = 10 * time.Millisecond
+ if _, _, err := f.host.beginStream(context.Background()); !errors.Is(err, errTooManyCalls) {
+ t.Fatalf("a fetch past streamWait = %v, want errTooManyCalls", err)
+ }
+ if free := cap(f.host.streams) - len(f.host.streams); free != 0 {
+ t.Fatalf("%d windows free after the waits gave up, want 0 (one held)", free)
+ }
+}
+
// TestHostCancelAfterHeadStopsTheFetch: a context that ends after the head,
// with nothing reading or closing the body, still stops the upstream fetch.
func TestHostCancelAfterHeadStopsTheFetch(t *testing.T) {