Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a turn held before routing shows on Routing and is answered 504 after 2m
yetone07936ca8906a
diff --git a/docs/subsystems/gateway-routing.md b/docs/subsystems/gateway-routing.mdindex f3ca67c03..1022e27b3 100644--- a/docs/subsystems/gateway-routing.md+++ b/docs/subsystems/gateway-routing.md@@ -231,6 +231,21 @@ recorded as a prediction failure (`noPrediction`).has in flight. `TestHandedOverGatewayFinishesItsStream`,`TestInheritedForAnotherAddressIsLetGo`, `TestToolCallOfTheMagpieBeforeGoesToIt` in[`handover_test.go`](../../internal/gateway/handover_test.go).+- An agent's turn (Chat, Responses, Messages, Gemini's generateContent and+ Codex's backend `/responses`) that magpie holds before any route is begun+ for it is watched in `counted` ([`unrouted.go`](../../internal/gateway/unrouted.go);+ heyusaku6 on X: Codex thought for over ten minutes with the Routing page+ empty, until Codex and magpie were restarted; what held it wasn't+ found). After `unroutedShow` (15s) a route of provider `magpie` shows it+ on the Routing page as routing, and gives way to the request's own route+ once one is begun (`noteRoute`, `trace.drop`). After `unroutedMost` (2m)+ with no route and nothing written, it is answered 504 in its client's+ protocol, whole (`Content-Length`, `Connection: close`) so the client+ has it while the handler may still be held; its context ends with+ `errUnrouted`, its route ends with the error, and the log has every+ goroutine's stack (at most once in 10 minutes). The handler's writes go+ through `heldWriter`, and nowhere once magpie has answered for it.+ `TestUnroutedRequestShownThenAnswered`, `TestUnroutedShownGivesWayToItsRoute`.## Remote magpie APIsdiff --git a/internal/gateway/active.go b/internal/gateway/active.goindex 8716b542a..8eeee6103 100644--- a/internal/gateway/active.go+++ b/internal/gateway/active.go@@ -42,6 +42,8 @@ type inflight struct {// begun: every route begun for it, the first its own, for endBrokenmu sync.Mutexbegun []*Route+ // unrouted: its watch while no route is begun for it (unrouted.go)+ unrouted *unroutedHold}type inflightKey struct{}@@ -99,7 +101,15 @@ func noteRoute(ctx context.Context, rt *Route) {f.route.CompareAndSwap(nil, rt)f.mu.Lock()f.begun = append(f.begun, rt)+ var shown *Route+ if f.unrouted != nil {+ shown = f.unrouted.routed()+ }f.mu.Unlock()+ if shown != nil {+ // the route that showed it while it waited gives way to its own+ f.unrouted.s.trace.drop(shown)+ }}}diff --git a/internal/gateway/busy.go b/internal/gateway/busy.goindex b2edb7804..286b5a526 100644--- a/internal/gateway/busy.go+++ b/internal/gateway/busy.go@@ -65,6 +65,7 @@ func (s *Server) counted(h http.Handler) http.Handler {}s.inFlight.Add(1)r, done := s.active.add(r)+ w, r, unwatch := s.watchUnrouted(w, r, inflightOf(r.Context()))defer func() {// a panic is net/http's to recover, as before; the routes it// left in flight end first (#2386)@@ -72,6 +73,7 @@ func (s *Server) counted(h http.Handler) http.Handler {s.endBroken(inflightOf(r.Context()))defer panic(p)}+ unwatch()done()s.lastDone.Store(time.Now().UnixNano())if s.inFlight.Add(-1) == 0 {diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.goindex 390186b3e..95a5c70f0 100644--- a/internal/gateway/gateway.go+++ b/internal/gateway/gateway.go@@ -1470,6 +1470,7 @@ func (s *Server) handle(from provider.Protocol) http.HandlerFunc {writeError(w, from, 400, err.Error())return}+ noteAsked(r.Context(), model)// Codex with magpie as its provider: spawn_agent names every// model a subagent may be put on, not only Codex's first five// (Gearclickist on Discord)@@ -1564,6 +1565,7 @@ func (s *Server) gemini(w http.ResponseWriter, r *http.Request) {writeError(w, provider.Gemini, 400, err.Error())return}+ noteAsked(r.Context(), model)var stream boolswitch method {case "generateContent":diff --git a/internal/gateway/trace.go b/internal/gateway/trace.goindex 0b6389659..e166e63b3 100644--- a/internal/gateway/trace.go+++ b/internal/gateway/trace.go@@ -663,6 +663,17 @@ func (t *trace) trim() {}}+// drop takes rt off the trace, a route that only stood for a request+// until its own began (unrouted.go).+func (t *trace) drop(rt *Route) {+ t.mu.Lock()+ defer t.mu.Unlock()+ if i := slices.Index(t.routes, rt); i >= 0 {+ t.routes = slices.Delete(t.routes, i, i+1)+ t.changed()+ }+}+// update changes a route under the lock.// maskedOf is r's Masked, read under the lock.func (t *trace) maskedOf(r *Route) *Masked {diff --git a/internal/gateway/unrouted.go b/internal/gateway/unrouted.gonew file mode 100644index 000000000..456b62eb9--- /dev/null+++ b/internal/gateway/unrouted.go@@ -0,0 +1,313 @@+package gateway++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "log"+ "net/http"+ "net/http/httptest"+ "runtime/pprof"+ "strconv"+ "strings"+ "sync"+ "sync/atomic"+ "time"++ "github.com/yetone/magpie/internal/access"+ "github.com/yetone/magpie/internal/provider"+)++// An agent's request that magpie holds before routing it went nowhere a+// user could see (heyusaku6 on X): Codex thought for over ten minutes on a+// routing group's model, then on two others, the Routing page showed none+// of them, and only a restart of Codex and magpie ended it. Codex waits for+// a reply's headers with no limit of its own, and a request shows on the+// Routing page once a route is begun for it, so whatever held it before+// that (a lock, a wait with no end) held it in silence.+//+// What held it isn't known. So a request with no route begun for it is+// shown on the Routing page as routing once it has waited unroutedShow,+// and is answered with an error once it has waited unroutedMost, with+// magpie's log saying where every goroutine stood. Its context ends then+// too, so what waits on it lets go.+var unroutedShow, unroutedMost = 15 * time.Second, 2 * time.Minute++// unroutedWatched is whether a request to path is an agent's turn, which+// is given a route once magpie knows where it goes: Chat, Responses,+// Messages, Gemini and Codex's own backend. A compaction or a model list+// has no route of its own, and an MCP or a search call is not a turn.+func unroutedWatched(path string) bool {+ switch path {+ case "/v1/chat/completions", "/chat/completions", "/v1/responses", "/responses",+ "/v1/messages", "/messages", CodexPath + "/responses":+ return true+ }+ return strings.HasPrefix(path, "/v1beta/models/") && (strings.HasSuffix(path, ":generateContent") || strings.HasSuffix(path, ":streamGenerateContent"))+}++// unroutedProto is the API path's client speaks, for its error.+func unroutedProto(path string) provider.Protocol {+ switch {+ case strings.HasSuffix(path, "/messages"):+ return provider.Anthropic+ case strings.HasPrefix(path, "/v1beta/"):+ return provider.Gemini+ case strings.HasSuffix(path, "/responses"):+ return provider.Responses+ }+ return provider.Chat+}++// errUnrouted ends a request's context when magpie stopped holding it.+var errUnrouted = errors.New("magpie held the request too long before routing it")++// heldWriter is the client's writer while magpie may still answer for its+// handler: what the handler writes goes through it, and once magpie has+// answered (gaveUp) the handler's writes go nowhere. The handler's headers+// are its own until it writes, so magpie's answer never shares a map+// another goroutine is filling.+type heldWriter struct {+ w http.ResponseWriter+ mu sync.Mutex+ hdr http.Header+ wrote bool // the handler began its reply+ gaveUp bool // magpie answered for it+}++func newHeldWriter(w http.ResponseWriter) *heldWriter {+ return &heldWriter{w: w, hdr: http.Header{}}+}++func (h *heldWriter) Header() http.Header {+ h.mu.Lock()+ defer h.mu.Unlock()+ if h.wrote {+ return h.w.Header()+ }+ return h.hdr+}++// begin is the handler's reply starting: its headers go to the client's.+// Called with mu held; false once magpie has answered.+func (h *heldWriter) begin() bool {+ if h.gaveUp {+ return false+ }+ if !h.wrote {+ h.wrote = true+ out := h.w.Header()+ for k, vs := range h.hdr {+ out[k] = vs+ }+ }+ return true+}++func (h *heldWriter) WriteHeader(code int) {+ h.mu.Lock()+ defer h.mu.Unlock()+ if h.begin() {+ h.w.WriteHeader(code)+ }+}++func (h *heldWriter) Write(b []byte) (int, error) {+ h.mu.Lock()+ defer h.mu.Unlock()+ if !h.begin() {+ return 0, http.ErrHandlerTimeout+ }+ return h.w.Write(b)+}++func (h *heldWriter) Flush() { _ = h.FlushError() }++// FlushError is http.ResponseController's Flush, which looks for it+// before it unwraps.+func (h *heldWriter) FlushError() error {+ h.mu.Lock()+ defer h.mu.Unlock()+ if !h.begin() {+ return http.ErrHandlerTimeout+ }+ return http.NewResponseController(h.w).Flush()+}++// Unwrap is for http.ResponseController's deadlines.+func (h *heldWriter) Unwrap() http.ResponseWriter { return h.w }++// answer writes magpie's own reply, whole, unless the handler began one.+// Its length is said, so the client has all of it while the handler may+// still be held: a reply of no length ends only when the handler does.+func (h *heldWriter) answer(proto provider.Protocol, status int, msg string) bool {+ h.mu.Lock()+ defer h.mu.Unlock()+ if h.wrote || h.gaveUp {+ return false+ }+ h.gaveUp = true+ rec := httptest.NewRecorder()+ writeError(rec, proto, status, msg)+ out := h.w.Header()+ for k, vs := range rec.Header() {+ out[k] = vs+ }+ out.Set("Content-Length", strconv.Itoa(rec.Body.Len()))+ out.Set("Connection", "close")+ h.w.WriteHeader(status)+ h.w.Write(rec.Body.Bytes())+ _ = http.NewResponseController(h.w).Flush()+ return true+}++// unroutedHold is one watched request's watch: the route that shows it+// while it waits, and what ends it.+type unroutedHold struct {+ s *Server+ r *http.Request+ f *inflight+ w *heldWriter+ cancel context.CancelCauseFunc+ asked atomic.Value // string: the model it asked for, once read+ shown *Route // under f.mu+ ended bool // under f.mu: answered by magpie+ over bool // under f.mu: its handler returned+}++type unroutedKey struct{}++// noteAsked tells ctx's watch the model its request asked for, for the+// route that shows it.+func noteAsked(ctx context.Context, model string) {+ if u, _ := ctx.Value(unroutedKey{}).(*unroutedHold); u != nil {+ u.asked.Store(model)+ }+}++// watchUnrouted watches r, counted as f, until the func it answers is+// called as its handler returns.+func (s *Server) watchUnrouted(w http.ResponseWriter, r *http.Request, f *inflight) (http.ResponseWriter, *http.Request, func()) {+ if f == nil || !unroutedWatched(r.URL.Path) {+ return w, r, func() {}+ }+ ctx, cancel := context.WithCancelCause(r.Context())+ u := &unroutedHold{s: s, f: f, w: newHeldWriter(w), cancel: cancel}+ r = r.WithContext(context.WithValue(ctx, unroutedKey{}, u))+ u.r = r+ f.mu.Lock()+ f.unrouted = u+ f.mu.Unlock()+ show := time.AfterFunc(unroutedShow, u.show)+ most := time.AfterFunc(unroutedMost, u.giveUp)+ return u.w, r, func() {+ show.Stop()+ most.Stop()+ f.mu.Lock()+ // a timer that fired as the handler returned does nothing now+ u.over = true+ shown := u.shown+ if !u.ended {+ u.shown = nil+ }+ f.mu.Unlock()+ if shown != nil && !u.ended {+ // it was answered with no route of its own: it is the+ // usage log's, as before+ s.trace.drop(shown)+ }+ cancel(context.Canceled)+ }+}++// show begins the route that shows the request on the Routing page, as+// routing, while nothing else does.+func (u *unroutedHold) show() {+ f := u.f+ f.mu.Lock()+ defer f.mu.Unlock()+ if len(f.begun) > 0 || u.shown != nil || u.ended || u.over {+ return+ }+ u.showLocked()+}++// showLocked begins the route that shows it; f.mu is held.+func (u *unroutedHold) showLocked() {+ asked, _ := u.asked.Load().(string)+ rt := Route{Time: u.f.start, Agent: callerOf(u.r).agent, Session: u.f.session, Model: asked, Provider: "magpie"}+ if rt.Agent == "" {+ rt.Agent = u.f.agent+ }+ rt.callerKey = access.Caller(u.r.Context()).KeyID+ u.shown = u.s.trace.begin(rt)+}++// routed is the request's own route begun: the one that showed it goes.+func (u *unroutedHold) routed() (drop *Route) {+ if u.ended {+ return nil+ }+ drop, u.shown = u.shown, nil+ return drop+}++// giveUp answers the request with an error, when no route was begun for it+// and its handler wrote nothing, ends its context, and says on its route+// and in the log what held it.+func (u *unroutedHold) giveUp() {+ f := u.f+ f.mu.Lock()+ if len(f.begun) > 0 || u.ended || u.over {+ f.mu.Unlock()+ return+ }+ held := time.Since(f.start).Round(time.Second)+ msg := fmt.Sprintf("magpie held this request for %s without routing it to any provider, so it was stopped rather than left waiting. Send it again; if it keeps happening, restart magpie and report it with magpie's log, which says where it was held.", held)+ if !u.w.answer(unroutedProto(u.r.URL.Path), http.StatusGatewayTimeout, msg) {+ f.mu.Unlock()+ return // the handler is answering+ }+ u.ended = true+ if u.shown == nil {+ u.showLocked()+ }+ shown := u.shown+ f.mu.Unlock()+ u.s.trace.update(shown, func(r *Route) {+ r.Done, r.Status, r.Error, r.Millis = true, http.StatusGatewayTimeout, msg, time.Since(r.Time).Milliseconds()+ })+ u.cancel(errUnrouted)+ asked, _ := u.asked.Load().(string)+ log.Printf("gateway: %s %s for %s from %s held %s before routing; stopped with 504%s", u.r.Method, u.r.URL.Path, cmpOr(asked, "?"), cmpOr(f.agent, "?"), held, unroutedStacks())+}++func cmpOr(a, b string) string {+ if a != "" {+ return a+ }+ return b+}++// unroutedDumped is when the goroutines were last written to the log for+// a held request: one dump says where, and a burst of held requests+// writes it once.+var unroutedDumped atomic.Int64++// unroutedStacks is every goroutine's stack, alike ones counted once, for+// the log at most once in ten minutes; "" otherwise.+func unroutedStacks() string {+ now := time.Now().UnixNano()+ last := unroutedDumped.Load()+ if last != 0 && now-last < int64(10*time.Minute) || !unroutedDumped.CompareAndSwap(last, now) {+ return ""+ }+ var b bytes.Buffer+ _ = pprof.Lookup("goroutine").WriteTo(&b, 1)+ const most = 256 << 10+ if b.Len() > most {+ b.Truncate(most)+ }+ return "; where every goroutine stood:\n" + b.String()+}diff --git a/internal/gateway/unrouted_test.go b/internal/gateway/unrouted_test.gonew file mode 100644index 000000000..418400c28--- /dev/null+++ b/internal/gateway/unrouted_test.go@@ -0,0 +1,177 @@+package gateway++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"+)++// unroutedLimits sets when a request held before routing shows, and when+// it is answered for.+func unroutedLimits(t *testing.T, show, most time.Duration) {+ t.Helper()+ s, m := unroutedShow, unroutedMost+ unroutedShow, unroutedMost = show, most+ t.Cleanup(func() { unroutedShow, unroutedMost = s, m })+}++// heldSight holds pastedImage before it is routed: its image's+// description is in flight and never ends, as a describe stuck on+// something that never lets go would be. What it holds the request on is+// returned, closed to let it go with text.+func heldSight(t *testing.T, s *Server) (release func(text string)) {+ t.Helper()+ model, ok := seer()+ if !ok {+ t.Fatal("no model to describe images with")+ }+ sum := sha256.Sum256([]byte(model + "\x00" + visionEffort(model) + "\x00" + "data:image/png;base64,aGVsbG8="))+ key := hex.EncodeToString(sum[:])+ e := &sight{done: make(chan struct{})}+ s.sightMu.Lock()+ if s.sights == nil {+ s.sights = map[string]*sight{}+ }+ s.sights[key] = e+ s.sightOrder = append(s.sightOrder, key)+ s.sightMu.Unlock()+ closed := false+ t.Cleanup(func() {+ if !closed {+ close(e.done)+ }+ })+ return func(text string) {+ e.text = text+ closed = true+ close(e.done)+ }+}++// routesOf is the trace's routes for model now.+func routesOf(s *Server, model string) []Route {+ var out []Route+ for _, r := range s.Trace(context.Background(), 0, 0).Routes {+ if r.Model == model {+ out = append(out, r)+ }+ }+ return out+}++// heyusaku6 on X: Codex thought for over ten minutes on a model served+// through magpie, the Routing page showed nothing, and only restarting+// both ended it. A request held before any route is begun for it shows on+// the Routing page as routing, and is answered with an error in bounded+// time, whole, rather than left to wait with nothing said.+func TestUnroutedRequestShownThenAnswered(t *testing.T) {+ unroutedLimits(t, 100*time.Millisecond, time.Second)+ s, _ := slowEyed(t, 0)+ heldSight(t, s)+ srv := httptest.NewServer(s.Handler())+ t.Cleanup(srv.Close)++ type reply struct {+ code int+ body string+ err error+ }+ got := make(chan reply, 1)+ go func() {+ c := &http.Client{Timeout: 20 * time.Second}+ resp, err := c.Post(srv.URL+"/v1/chat/completions", "application/json", strings.NewReader(pastedImage))+ if err != nil {+ got <- reply{err: err}+ return+ }+ b, err := io.ReadAll(resp.Body)+ resp.Body.Close()+ got <- reply{resp.StatusCode, string(b), err}+ }()++ // shown as routing while it waits+ deadline := time.Now().Add(10 * time.Second)+ for {+ rs := routesOf(s, "probe/text")+ if len(rs) == 1 && !rs[0].Done {+ break+ }+ if time.Now().After(deadline) {+ t.Fatalf("the held request never showed on the Routing page: %+v", rs)+ }+ time.Sleep(10 * time.Millisecond)+ }++ var r reply+ select {+ case r = <-got:+ case <-time.After(15 * time.Second):+ t.Fatal("the held request was never answered")+ }+ if r.err != nil || r.code != http.StatusGatewayTimeout || !strings.Contains(r.body, "without routing it to any provider") {+ t.Fatalf("held request answered %d %q %v", r.code, r.body, r.err)+ }+ rs := routesOf(s, "probe/text")+ if len(rs) != 1 || !rs[0].Done || rs[0].Status != http.StatusGatewayTimeout || !strings.Contains(rs[0].Error, "without routing") {+ t.Fatalf("the held request's route: %+v", rs)+ }+}++// One shown while it waited gives way to its own route once routed: the+// Routing page has the request once, and its reply is the provider's.+func TestUnroutedShownGivesWayToItsRoute(t *testing.T) {+ unroutedLimits(t, 50*time.Millisecond, time.Minute)+ s, u := slowEyed(t, 0)+ u.open()+ release := heldSight(t, s)+ done := make(chan int, 1)+ go func() {+ code, _ := postAs(t, s, "", pastedImage)+ done <- code+ }()+ deadline := time.Now().Add(10 * time.Second)+ for len(routesOf(s, "probe/text")) == 0 {+ if time.Now().After(deadline) {+ t.Fatal("the held request never showed on the Routing page")+ }+ time.Sleep(10 * time.Millisecond)+ }+ release("HELLO")+ select {+ case code := <-done:+ if code != 200 {+ t.Fatalf("routed after its wait: %d", code)+ }+ case <-time.After(15 * time.Second):+ t.Fatal("never answered once let go")+ }+ rs := routesOf(s, "probe/text")+ if len(rs) != 1 || rs[0].Provider != "probe" || rs[0].Status != 200 {+ t.Fatalf("routes for the request: %+v", rs)+ }+}++// A request answered before it was shown leaves nothing on the trace of+// its wait, and a timer that fires as its handler returns adds nothing.+func TestUnroutedQuickRequestLeavesNoTrace(t *testing.T) {+ unroutedLimits(t, time.Millisecond, time.Minute)+ s, _ := slowEyed(t, 0)+ for range 20 {+ code, _ := postAs(t, s, "", `{"model":"nowhere/none","messages":[{"role":"user","content":"hi"}]}`)+ if code < 400 {+ t.Fatalf("an unknown model answered %d", code)+ }+ }+ time.Sleep(50 * time.Millisecond)+ for _, r := range s.Trace(context.Background(), 0, 0).Routes {+ if r.Provider == "magpie" && !r.Done {+ t.Fatalf("a route left showing a request that was answered: %+v", r)+ }+ }+}