Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
plugin: a fetch past the host's 128 streams waits for one, not a local 502 (#3890)
yetonef8cd964aec76
diff --git a/internal/gateway/plugin_stream_admit_test.go b/internal/gateway/plugin_stream_admit_test.gonew file mode 100644index 000000000..eeb730f15--- /dev/null+++ b/internal/gateway/plugin_stream_admit_test.go@@ -0,0 +1,103 @@+package gateway++import (+ "context"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++// #3890: 192 streams at once through one plugin. The plugin host holds 128+// streams' windows; the rest used to be refused locally at once ("too many+// plugin calls are in flight"), each refusal a 502 try the gateway sent+// again. Now they wait for a window: every request is answered on its+// first try, and the vendor sees each once.+func TestPluginStreamsPastTheBudgetWait(t *testing.T) {+ const n = 192+ var hits, live, most atomic.Int64+ release := make(chan struct{})+ pid := besideFake(t, "stream-admit", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ io.Copy(io.Discard, r.Body)+ hits.Add(1)+ now := live.Add(1)+ defer live.Add(-1)+ for m := most.Load(); now > m && !most.CompareAndSwap(m, now); m = most.Load() {+ }+ w.Header().Set("Content-Type", "text/event-stream")+ w.WriteHeader(200)+ w.(http.Flusher).Flush()+ select {+ case <-release:+ case <-r.Context().Done():+ return+ }+ io.WriteString(w, `data: {"id":"a","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}`+"\n\n")+ io.WriteString(w, `data: {"id":"a","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}`+"\n\n")+ io.WriteString(w, "data: [DONE]\n\n")+ }))+ s := New()+ gw := httptest.NewServer(s.Handler())+ t.Cleanup(gw.Close)+ tr := http.DefaultTransport.(*http.Transport).Clone()+ tr.Proxy, tr.MaxIdleConnsPerHost = nil, n+ t.Cleanup(tr.CloseIdleConnections)+ client := &http.Client{Transport: tr, Timeout: 2 * time.Minute}++ model := pid + "/fake-1"+ var wg sync.WaitGroup+ errs := make(chan string, n)+ for range n {+ wg.Add(1)+ go func() {+ defer wg.Done()+ res, err := client.Post(gw.URL+"/v1/chat/completions", "application/json", strings.NewReader(pluginReplyRequest(model, true)))+ if err != nil {+ errs <- err.Error()+ return+ }+ b, _ := io.ReadAll(res.Body)+ res.Body.Close()+ if res.StatusCode != 200 || !strings.Contains(string(b), "ok") {+ errs <- res.Status + " " + string(b)+ }+ }()+ }+ // the host's windows are all held by streams the vendor keeps open+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute)+ defer cancel()+ for live.Load() < 128 && ctx.Err() == nil {+ time.Sleep(10 * time.Millisecond)+ }+ if live.Load() < 128 {+ t.Fatalf("the vendor holds %d streams, want the host's 128", live.Load())+ }+ time.Sleep(300 * time.Millisecond) // the other 64 reach the host meanwhile+ close(release)+ wg.Wait()+ close(errs)+ for e := range errs {+ t.Error(e)+ }+ if h := hits.Load(); h != n {+ t.Errorf("the vendor saw %d requests, want %d", h, n)+ }+ if m := most.Load(); m > 128 {+ t.Errorf("the vendor held %d streams at once, past the host's 128", m)+ }+ routes := routesOf(s, model)+ if len(routes) == 0 {+ t.Fatal("no routes traced")+ }+ for _, r := range routes {+ if len(r.Tries) != 1 || r.Tries[0].Status != 200 {+ for _, tr := range r.Tries {+ t.Errorf("route %d: a try %d %s", r.ID, tr.Status, tr.Error)+ }+ }+ }+}diff --git a/internal/plugin/host.go b/internal/plugin/host.goindex 6cf8b05f1..5b311a39e 100644--- a/internal/plugin/host.go+++ b/internal/plugin/host.go@@ -153,6 +153,13 @@ var (// under it, and so is any RPC result, so a child that never ends a line// fails the host rather than making it allocate without bound.maxWireLine = 32 << 20+ // streamWait is how long a fetch waits for a stream's window to come+ // free when the budget's are all held, while its request lives. Past+ // the budget a fetch used to fail at once (errTooManyCalls), and the+ // gateway counted that as a 502 and tried again: 192 streams at once+ // through one plugin gave 298 local refusals that never reached the+ // vendor (#3890).+ streamWait = time.Minute// maxHeadBytes and maxHeadEntries are magpie's own head-envelope policy,// not HTTP's: a fetch's head carries at most maxHeadBytes of header bytes// (the UTF-8 bytes of every name and value, plus headEntryCost an entry)@@ -1445,14 +1452,7 @@ func (h *host) dropCtrl() {// when the host already holds its limit of live and retiring calls, so a new// call fails clearly rather than a legitimate accepted reply being cut short.func (h *host) begin(stream bool) (int64, *call, error) {- h.mu.Lock()- defer h.mu.Unlock()- if !h.alive() {- return 0, nil, errHostGone- }- c := &call{done: make(chan message, 1), stream: stream, wake: make(chan struct{}, 1), state: callSending}if stream {- c.window = streamWindow// a stream holds one window of the budget as well as a call slot, so// the RPC reserve is never spent on one more streamselect {@@ -1461,6 +1461,46 @@ func (h *host) begin(stream bool) (int64, *call, error) {return 0, nil, errTooManyCalls}}+ return h.admit(stream)+}++// beginStream is begin for a fetch: while the budget's windows are all held+// it waits for one to come free, for streamWait at most and only while ctx+// lives, rather than failing a request that would have run a moment later.+// The wait is outside h.mu, which the call that frees a window takes.+func (h *host) beginStream(ctx context.Context) (int64, *call, error) {+ select {+ case h.streams <- struct{}{}:+ default:+ t := time.NewTimer(streamWait)+ defer t.Stop()+ select {+ case h.streams <- struct{}{}:+ case <-ctx.Done():+ return 0, nil, context.Cause(ctx)+ case <-h.dead:+ return 0, nil, errHostGone+ case <-t.C:+ return 0, nil, errTooManyCalls+ }+ }+ return h.admit(true)+}++// admit registers a call whose stream window, if it is a stream, is taken.+func (h *host) admit(stream bool) (int64, *call, error) {+ h.mu.Lock()+ defer h.mu.Unlock()+ if !h.alive() {+ if stream {+ <-h.streams+ }+ return 0, nil, errHostGone+ }+ c := &call{done: make(chan message, 1), stream: stream, wake: make(chan struct{}, 1), state: callSending}+ if stream {+ c.window = streamWindow+ }select {case h.slots <- struct{}{}:default:@@ -1693,7 +1733,7 @@ func Fetch(ctx context.Context, r FetchRequest) (*http.Response, error) {if r.Proxy == "" {r.Proxy = proxyOf(ctx)}- id, c, err := h.begin(true)+ id, c, err := h.beginStream(ctx)if err != nil {return nil, err}diff --git a/internal/plugin/host_test.go b/internal/plugin/host_test.goindex 51daeca51..bb2669e3f 100644--- a/internal/plugin/host_test.go+++ b/internal/plugin/host_test.go@@ -675,6 +675,62 @@ func TestHostAdmissionLimitFailsTheNewCall(t *testing.T) {}}+// TestHostFetchWaitsForAStreamWindow (#3890): a fetch past the budget waits+// for a window to come free and then runs, rather than failing at once into+// a local 502 the gateway tries again; it gives up when its request ends or+// after streamWait.+func TestHostFetchWaitsForAStreamWindow(t *testing.T) {+ oldWindow, oldBudget := streamWindow, streamBudget+ oldReserve, oldWait := rpcReserve, streamWait+ streamWindow, streamBudget, rpcReserve = 1<<20, 1<<20, 0 // one streaming call+ t.Cleanup(func() { streamWindow, streamBudget, rpcReserve, streamWait = oldWindow, oldBudget, oldReserve, oldWait })++ f := newFakeHost(t)+ first, _, err := f.host.begin(true)+ if err != nil {+ t.Fatal(err)+ }+ type got struct {+ id int64+ err error+ }+ waited := make(chan got, 1)+ go func() {+ id, _, err := f.host.beginStream(context.Background())+ waited <- got{id, err}+ }()+ select {+ case g := <-waited:+ t.Fatalf("a fetch past the budget = %d, %v, want it waiting", g.id, g.err)+ case <-time.After(50 * time.Millisecond):+ }+ f.host.mu.Lock()+ c := f.host.calls[first]+ f.host.mu.Unlock()+ c.release()+ if g := <-waited; g.err != nil {+ t.Fatalf("the waiting fetch, once a window came free: %v", g.err)+ }++ // the budget is held again: a request that ends gives the wait up+ ctx, cancel := context.WithCancel(context.Background())+ go func() {+ _, _, err := f.host.beginStream(ctx)+ waited <- got{0, err}+ }()+ cancel()+ if g := <-waited; !errors.Is(g.err, context.Canceled) {+ t.Fatalf("a fetch whose request ended = %v, want canceled", g.err)+ }+ streamWait = 10 * time.Millisecond+ if _, _, err := f.host.beginStream(context.Background()); !errors.Is(err, errTooManyCalls) {+ t.Fatalf("a fetch past streamWait = %v, want errTooManyCalls", err)+ }+ if free := cap(f.host.streams) - len(f.host.streams); free != 0 {+ t.Fatalf("%d windows free after the waits gave up, want 0 (one held)", free)+ }+}+// TestHostCancelAfterHeadStopsTheFetch: a context that ends after the head,// with nothing reading or closing the body, still stops the upstream fetch.func TestHostCancelAfterHeadStopsTheFetch(t *testing.T) {