Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a turn held before routing shows on Routing and is answered 504 after 2m

yetone07936ca8906a
diff --git a/docs/subsystems/gateway-routing.md b/docs/subsystems/gateway-routing.md
index f3ca67c03..1022e27b3 100644
--- a/docs/subsystems/gateway-routing.md
+++ b/docs/subsystems/gateway-routing.md
@@ -231,6 +231,21 @@ recorded as a prediction failure (`noPrediction`).
has in flight. `TestHandedOverGatewayFinishesItsStream`,
`TestInheritedForAnotherAddressIsLetGo`, `TestToolCallOfTheMagpieBeforeGoesToIt` in
[`handover_test.go`](../../internal/gateway/handover_test.go).
+- An agent's turn (Chat, Responses, Messages, Gemini's generateContent and
+ Codex's backend `/responses`) that magpie holds before any route is begun
+ for it is watched in `counted` ([`unrouted.go`](../../internal/gateway/unrouted.go);
+ heyusaku6 on X: Codex thought for over ten minutes with the Routing page
+ empty, until Codex and magpie were restarted; what held it wasn't
+ found). After `unroutedShow` (15s) a route of provider `magpie` shows it
+ on the Routing page as routing, and gives way to the request's own route
+ once one is begun (`noteRoute`, `trace.drop`). After `unroutedMost` (2m)
+ with no route and nothing written, it is answered 504 in its client's
+ protocol, whole (`Content-Length`, `Connection: close`) so the client
+ has it while the handler may still be held; its context ends with
+ `errUnrouted`, its route ends with the error, and the log has every
+ goroutine's stack (at most once in 10 minutes). The handler's writes go
+ through `heldWriter`, and nowhere once magpie has answered for it.
+ `TestUnroutedRequestShownThenAnswered`, `TestUnroutedShownGivesWayToItsRoute`.
## Remote magpie APIs
diff --git a/internal/gateway/active.go b/internal/gateway/active.go
index 8716b542a..8eeee6103 100644
--- a/internal/gateway/active.go
+++ b/internal/gateway/active.go
@@ -42,6 +42,8 @@ type inflight struct {
// begun: every route begun for it, the first its own, for endBroken
mu sync.Mutex
begun []*Route
+ // unrouted: its watch while no route is begun for it (unrouted.go)
+ unrouted *unroutedHold
}
type inflightKey struct{}
@@ -99,7 +101,15 @@ func noteRoute(ctx context.Context, rt *Route) {
f.route.CompareAndSwap(nil, rt)
f.mu.Lock()
f.begun = append(f.begun, rt)
+ var shown *Route
+ if f.unrouted != nil {
+ shown = f.unrouted.routed()
+ }
f.mu.Unlock()
+ if shown != nil {
+ // the route that showed it while it waited gives way to its own
+ f.unrouted.s.trace.drop(shown)
+ }
}
}
diff --git a/internal/gateway/busy.go b/internal/gateway/busy.go
index b2edb7804..286b5a526 100644
--- a/internal/gateway/busy.go
+++ b/internal/gateway/busy.go
@@ -65,6 +65,7 @@ func (s *Server) counted(h http.Handler) http.Handler {
}
s.inFlight.Add(1)
r, done := s.active.add(r)
+ w, r, unwatch := s.watchUnrouted(w, r, inflightOf(r.Context()))
defer func() {
// a panic is net/http's to recover, as before; the routes it
// left in flight end first (#2386)
@@ -72,6 +73,7 @@ func (s *Server) counted(h http.Handler) http.Handler {
s.endBroken(inflightOf(r.Context()))
defer panic(p)
}
+ unwatch()
done()
s.lastDone.Store(time.Now().UnixNano())
if s.inFlight.Add(-1) == 0 {
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go
index 390186b3e..95a5c70f0 100644
--- a/internal/gateway/gateway.go
+++ b/internal/gateway/gateway.go
@@ -1470,6 +1470,7 @@ func (s *Server) handle(from provider.Protocol) http.HandlerFunc {
writeError(w, from, 400, err.Error())
return
}
+ noteAsked(r.Context(), model)
// Codex with magpie as its provider: spawn_agent names every
// model a subagent may be put on, not only Codex's first five
// (Gearclickist on Discord)
@@ -1564,6 +1565,7 @@ func (s *Server) gemini(w http.ResponseWriter, r *http.Request) {
writeError(w, provider.Gemini, 400, err.Error())
return
}
+ noteAsked(r.Context(), model)
var stream bool
switch method {
case "generateContent":
diff --git a/internal/gateway/trace.go b/internal/gateway/trace.go
index 0b6389659..e166e63b3 100644
--- a/internal/gateway/trace.go
+++ b/internal/gateway/trace.go
@@ -663,6 +663,17 @@ func (t *trace) trim() {
}
}
+// drop takes rt off the trace, a route that only stood for a request
+// until its own began (unrouted.go).
+func (t *trace) drop(rt *Route) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+ if i := slices.Index(t.routes, rt); i >= 0 {
+ t.routes = slices.Delete(t.routes, i, i+1)
+ t.changed()
+ }
+}
+
// update changes a route under the lock.
// maskedOf is r's Masked, read under the lock.
func (t *trace) maskedOf(r *Route) *Masked {
diff --git a/internal/gateway/unrouted.go b/internal/gateway/unrouted.go
new file mode 100644
index 000000000..456b62eb9
--- /dev/null
+++ b/internal/gateway/unrouted.go
@@ -0,0 +1,313 @@
+package gateway
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "net/http"
+ "net/http/httptest"
+ "runtime/pprof"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/yetone/magpie/internal/access"
+ "github.com/yetone/magpie/internal/provider"
+)
+
+// An agent's request that magpie holds before routing it went nowhere a
+// user could see (heyusaku6 on X): Codex thought for over ten minutes on a
+// routing group's model, then on two others, the Routing page showed none
+// of them, and only a restart of Codex and magpie ended it. Codex waits for
+// a reply's headers with no limit of its own, and a request shows on the
+// Routing page once a route is begun for it, so whatever held it before
+// that (a lock, a wait with no end) held it in silence.
+//
+// What held it isn't known. So a request with no route begun for it is
+// shown on the Routing page as routing once it has waited unroutedShow,
+// and is answered with an error once it has waited unroutedMost, with
+// magpie's log saying where every goroutine stood. Its context ends then
+// too, so what waits on it lets go.
+var unroutedShow, unroutedMost = 15 * time.Second, 2 * time.Minute
+
+// unroutedWatched is whether a request to path is an agent's turn, which
+// is given a route once magpie knows where it goes: Chat, Responses,
+// Messages, Gemini and Codex's own backend. A compaction or a model list
+// has no route of its own, and an MCP or a search call is not a turn.
+func unroutedWatched(path string) bool {
+ switch path {
+ case "/v1/chat/completions", "/chat/completions", "/v1/responses", "/responses",
+ "/v1/messages", "/messages", CodexPath + "/responses":
+ return true
+ }
+ return strings.HasPrefix(path, "/v1beta/models/") && (strings.HasSuffix(path, ":generateContent") || strings.HasSuffix(path, ":streamGenerateContent"))
+}
+
+// unroutedProto is the API path's client speaks, for its error.
+func unroutedProto(path string) provider.Protocol {
+ switch {
+ case strings.HasSuffix(path, "/messages"):
+ return provider.Anthropic
+ case strings.HasPrefix(path, "/v1beta/"):
+ return provider.Gemini
+ case strings.HasSuffix(path, "/responses"):
+ return provider.Responses
+ }
+ return provider.Chat
+}
+
+// errUnrouted ends a request's context when magpie stopped holding it.
+var errUnrouted = errors.New("magpie held the request too long before routing it")
+
+// heldWriter is the client's writer while magpie may still answer for its
+// handler: what the handler writes goes through it, and once magpie has
+// answered (gaveUp) the handler's writes go nowhere. The handler's headers
+// are its own until it writes, so magpie's answer never shares a map
+// another goroutine is filling.
+type heldWriter struct {
+ w http.ResponseWriter
+ mu sync.Mutex
+ hdr http.Header
+ wrote bool // the handler began its reply
+ gaveUp bool // magpie answered for it
+}
+
+func newHeldWriter(w http.ResponseWriter) *heldWriter {
+ return &heldWriter{w: w, hdr: http.Header{}}
+}
+
+func (h *heldWriter) Header() http.Header {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if h.wrote {
+ return h.w.Header()
+ }
+ return h.hdr
+}
+
+// begin is the handler's reply starting: its headers go to the client's.
+// Called with mu held; false once magpie has answered.
+func (h *heldWriter) begin() bool {
+ if h.gaveUp {
+ return false
+ }
+ if !h.wrote {
+ h.wrote = true
+ out := h.w.Header()
+ for k, vs := range h.hdr {
+ out[k] = vs
+ }
+ }
+ return true
+}
+
+func (h *heldWriter) WriteHeader(code int) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if h.begin() {
+ h.w.WriteHeader(code)
+ }
+}
+
+func (h *heldWriter) Write(b []byte) (int, error) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if !h.begin() {
+ return 0, http.ErrHandlerTimeout
+ }
+ return h.w.Write(b)
+}
+
+func (h *heldWriter) Flush() { _ = h.FlushError() }
+
+// FlushError is http.ResponseController's Flush, which looks for it
+// before it unwraps.
+func (h *heldWriter) FlushError() error {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if !h.begin() {
+ return http.ErrHandlerTimeout
+ }
+ return http.NewResponseController(h.w).Flush()
+}
+
+// Unwrap is for http.ResponseController's deadlines.
+func (h *heldWriter) Unwrap() http.ResponseWriter { return h.w }
+
+// answer writes magpie's own reply, whole, unless the handler began one.
+// Its length is said, so the client has all of it while the handler may
+// still be held: a reply of no length ends only when the handler does.
+func (h *heldWriter) answer(proto provider.Protocol, status int, msg string) bool {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if h.wrote || h.gaveUp {
+ return false
+ }
+ h.gaveUp = true
+ rec := httptest.NewRecorder()
+ writeError(rec, proto, status, msg)
+ out := h.w.Header()
+ for k, vs := range rec.Header() {
+ out[k] = vs
+ }
+ out.Set("Content-Length", strconv.Itoa(rec.Body.Len()))
+ out.Set("Connection", "close")
+ h.w.WriteHeader(status)
+ h.w.Write(rec.Body.Bytes())
+ _ = http.NewResponseController(h.w).Flush()
+ return true
+}
+
+// unroutedHold is one watched request's watch: the route that shows it
+// while it waits, and what ends it.
+type unroutedHold struct {
+ s *Server
+ r *http.Request
+ f *inflight
+ w *heldWriter
+ cancel context.CancelCauseFunc
+ asked atomic.Value // string: the model it asked for, once read
+ shown *Route // under f.mu
+ ended bool // under f.mu: answered by magpie
+ over bool // under f.mu: its handler returned
+}
+
+type unroutedKey struct{}
+
+// noteAsked tells ctx's watch the model its request asked for, for the
+// route that shows it.
+func noteAsked(ctx context.Context, model string) {
+ if u, _ := ctx.Value(unroutedKey{}).(*unroutedHold); u != nil {
+ u.asked.Store(model)
+ }
+}
+
+// watchUnrouted watches r, counted as f, until the func it answers is
+// called as its handler returns.
+func (s *Server) watchUnrouted(w http.ResponseWriter, r *http.Request, f *inflight) (http.ResponseWriter, *http.Request, func()) {
+ if f == nil || !unroutedWatched(r.URL.Path) {
+ return w, r, func() {}
+ }
+ ctx, cancel := context.WithCancelCause(r.Context())
+ u := &unroutedHold{s: s, f: f, w: newHeldWriter(w), cancel: cancel}
+ r = r.WithContext(context.WithValue(ctx, unroutedKey{}, u))
+ u.r = r
+ f.mu.Lock()
+ f.unrouted = u
+ f.mu.Unlock()
+ show := time.AfterFunc(unroutedShow, u.show)
+ most := time.AfterFunc(unroutedMost, u.giveUp)
+ return u.w, r, func() {
+ show.Stop()
+ most.Stop()
+ f.mu.Lock()
+ // a timer that fired as the handler returned does nothing now
+ u.over = true
+ shown := u.shown
+ if !u.ended {
+ u.shown = nil
+ }
+ f.mu.Unlock()
+ if shown != nil && !u.ended {
+ // it was answered with no route of its own: it is the
+ // usage log's, as before
+ s.trace.drop(shown)
+ }
+ cancel(context.Canceled)
+ }
+}
+
+// show begins the route that shows the request on the Routing page, as
+// routing, while nothing else does.
+func (u *unroutedHold) show() {
+ f := u.f
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if len(f.begun) > 0 || u.shown != nil || u.ended || u.over {
+ return
+ }
+ u.showLocked()
+}
+
+// showLocked begins the route that shows it; f.mu is held.
+func (u *unroutedHold) showLocked() {
+ asked, _ := u.asked.Load().(string)
+ rt := Route{Time: u.f.start, Agent: callerOf(u.r).agent, Session: u.f.session, Model: asked, Provider: "magpie"}
+ if rt.Agent == "" {
+ rt.Agent = u.f.agent
+ }
+ rt.callerKey = access.Caller(u.r.Context()).KeyID
+ u.shown = u.s.trace.begin(rt)
+}
+
+// routed is the request's own route begun: the one that showed it goes.
+func (u *unroutedHold) routed() (drop *Route) {
+ if u.ended {
+ return nil
+ }
+ drop, u.shown = u.shown, nil
+ return drop
+}
+
+// giveUp answers the request with an error, when no route was begun for it
+// and its handler wrote nothing, ends its context, and says on its route
+// and in the log what held it.
+func (u *unroutedHold) giveUp() {
+ f := u.f
+ f.mu.Lock()
+ if len(f.begun) > 0 || u.ended || u.over {
+ f.mu.Unlock()
+ return
+ }
+ held := time.Since(f.start).Round(time.Second)
+ msg := fmt.Sprintf("magpie held this request for %s without routing it to any provider, so it was stopped rather than left waiting. Send it again; if it keeps happening, restart magpie and report it with magpie's log, which says where it was held.", held)
+ if !u.w.answer(unroutedProto(u.r.URL.Path), http.StatusGatewayTimeout, msg) {
+ f.mu.Unlock()
+ return // the handler is answering
+ }
+ u.ended = true
+ if u.shown == nil {
+ u.showLocked()
+ }
+ shown := u.shown
+ f.mu.Unlock()
+ u.s.trace.update(shown, func(r *Route) {
+ r.Done, r.Status, r.Error, r.Millis = true, http.StatusGatewayTimeout, msg, time.Since(r.Time).Milliseconds()
+ })
+ u.cancel(errUnrouted)
+ asked, _ := u.asked.Load().(string)
+ log.Printf("gateway: %s %s for %s from %s held %s before routing; stopped with 504%s", u.r.Method, u.r.URL.Path, cmpOr(asked, "?"), cmpOr(f.agent, "?"), held, unroutedStacks())
+}
+
+func cmpOr(a, b string) string {
+ if a != "" {
+ return a
+ }
+ return b
+}
+
+// unroutedDumped is when the goroutines were last written to the log for
+// a held request: one dump says where, and a burst of held requests
+// writes it once.
+var unroutedDumped atomic.Int64
+
+// unroutedStacks is every goroutine's stack, alike ones counted once, for
+// the log at most once in ten minutes; "" otherwise.
+func unroutedStacks() string {
+ now := time.Now().UnixNano()
+ last := unroutedDumped.Load()
+ if last != 0 && now-last < int64(10*time.Minute) || !unroutedDumped.CompareAndSwap(last, now) {
+ return ""
+ }
+ var b bytes.Buffer
+ _ = pprof.Lookup("goroutine").WriteTo(&b, 1)
+ const most = 256 << 10
+ if b.Len() > most {
+ b.Truncate(most)
+ }
+ return "; where every goroutine stood:\n" + b.String()
+}
diff --git a/internal/gateway/unrouted_test.go b/internal/gateway/unrouted_test.go
new file mode 100644
index 000000000..418400c28
--- /dev/null
+++ b/internal/gateway/unrouted_test.go
@@ -0,0 +1,177 @@
+package gateway
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+)
+
+// unroutedLimits sets when a request held before routing shows, and when
+// it is answered for.
+func unroutedLimits(t *testing.T, show, most time.Duration) {
+ t.Helper()
+ s, m := unroutedShow, unroutedMost
+ unroutedShow, unroutedMost = show, most
+ t.Cleanup(func() { unroutedShow, unroutedMost = s, m })
+}
+
+// heldSight holds pastedImage before it is routed: its image's
+// description is in flight and never ends, as a describe stuck on
+// something that never lets go would be. What it holds the request on is
+// returned, closed to let it go with text.
+func heldSight(t *testing.T, s *Server) (release func(text string)) {
+ t.Helper()
+ model, ok := seer()
+ if !ok {
+ t.Fatal("no model to describe images with")
+ }
+ sum := sha256.Sum256([]byte(model + "\x00" + visionEffort(model) + "\x00" + "data:image/png;base64,aGVsbG8="))
+ key := hex.EncodeToString(sum[:])
+ e := &sight{done: make(chan struct{})}
+ s.sightMu.Lock()
+ if s.sights == nil {
+ s.sights = map[string]*sight{}
+ }
+ s.sights[key] = e
+ s.sightOrder = append(s.sightOrder, key)
+ s.sightMu.Unlock()
+ closed := false
+ t.Cleanup(func() {
+ if !closed {
+ close(e.done)
+ }
+ })
+ return func(text string) {
+ e.text = text
+ closed = true
+ close(e.done)
+ }
+}
+
+// routesOf is the trace's routes for model now.
+func routesOf(s *Server, model string) []Route {
+ var out []Route
+ for _, r := range s.Trace(context.Background(), 0, 0).Routes {
+ if r.Model == model {
+ out = append(out, r)
+ }
+ }
+ return out
+}
+
+// heyusaku6 on X: Codex thought for over ten minutes on a model served
+// through magpie, the Routing page showed nothing, and only restarting
+// both ended it. A request held before any route is begun for it shows on
+// the Routing page as routing, and is answered with an error in bounded
+// time, whole, rather than left to wait with nothing said.
+func TestUnroutedRequestShownThenAnswered(t *testing.T) {
+ unroutedLimits(t, 100*time.Millisecond, time.Second)
+ s, _ := slowEyed(t, 0)
+ heldSight(t, s)
+ srv := httptest.NewServer(s.Handler())
+ t.Cleanup(srv.Close)
+
+ type reply struct {
+ code int
+ body string
+ err error
+ }
+ got := make(chan reply, 1)
+ go func() {
+ c := &http.Client{Timeout: 20 * time.Second}
+ resp, err := c.Post(srv.URL+"/v1/chat/completions", "application/json", strings.NewReader(pastedImage))
+ if err != nil {
+ got <- reply{err: err}
+ return
+ }
+ b, err := io.ReadAll(resp.Body)
+ resp.Body.Close()
+ got <- reply{resp.StatusCode, string(b), err}
+ }()
+
+ // shown as routing while it waits
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ rs := routesOf(s, "probe/text")
+ if len(rs) == 1 && !rs[0].Done {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("the held request never showed on the Routing page: %+v", rs)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+
+ var r reply
+ select {
+ case r = <-got:
+ case <-time.After(15 * time.Second):
+ t.Fatal("the held request was never answered")
+ }
+ if r.err != nil || r.code != http.StatusGatewayTimeout || !strings.Contains(r.body, "without routing it to any provider") {
+ t.Fatalf("held request answered %d %q %v", r.code, r.body, r.err)
+ }
+ rs := routesOf(s, "probe/text")
+ if len(rs) != 1 || !rs[0].Done || rs[0].Status != http.StatusGatewayTimeout || !strings.Contains(rs[0].Error, "without routing") {
+ t.Fatalf("the held request's route: %+v", rs)
+ }
+}
+
+// One shown while it waited gives way to its own route once routed: the
+// Routing page has the request once, and its reply is the provider's.
+func TestUnroutedShownGivesWayToItsRoute(t *testing.T) {
+ unroutedLimits(t, 50*time.Millisecond, time.Minute)
+ s, u := slowEyed(t, 0)
+ u.open()
+ release := heldSight(t, s)
+ done := make(chan int, 1)
+ go func() {
+ code, _ := postAs(t, s, "", pastedImage)
+ done <- code
+ }()
+ deadline := time.Now().Add(10 * time.Second)
+ for len(routesOf(s, "probe/text")) == 0 {
+ if time.Now().After(deadline) {
+ t.Fatal("the held request never showed on the Routing page")
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ release("HELLO")
+ select {
+ case code := <-done:
+ if code != 200 {
+ t.Fatalf("routed after its wait: %d", code)
+ }
+ case <-time.After(15 * time.Second):
+ t.Fatal("never answered once let go")
+ }
+ rs := routesOf(s, "probe/text")
+ if len(rs) != 1 || rs[0].Provider != "probe" || rs[0].Status != 200 {
+ t.Fatalf("routes for the request: %+v", rs)
+ }
+}
+
+// A request answered before it was shown leaves nothing on the trace of
+// its wait, and a timer that fires as its handler returns adds nothing.
+func TestUnroutedQuickRequestLeavesNoTrace(t *testing.T) {
+ unroutedLimits(t, time.Millisecond, time.Minute)
+ s, _ := slowEyed(t, 0)
+ for range 20 {
+ code, _ := postAs(t, s, "", `{"model":"nowhere/none","messages":[{"role":"user","content":"hi"}]}`)
+ if code < 400 {
+ t.Fatalf("an unknown model answered %d", code)
+ }
+ }
+ time.Sleep(50 * time.Millisecond)
+ for _, r := range s.Trace(context.Background(), 0, 0).Routes {
+ if r.Provider == "magpie" && !r.Done {
+ t.Fatalf("a route left showing a request that was answered: %+v", r)
+ }
+ }
+}