Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a request held 30s before routing in no known wait is answered then

yetone5a94eec3aead
diff --git a/docs/subsystems/gateway-routing.md b/docs/subsystems/gateway-routing.md
index 3afcdcc39..f07978386 100644
--- a/docs/subsystems/gateway-routing.md
+++ b/docs/subsystems/gateway-routing.md
@@ -247,14 +247,27 @@ recorded as a prediction failure (`noPrediction`).
empty, until Codex and magpie were restarted; what held it wasn't
found). After `unroutedShow` (15s) a route of provider `magpie` shows it
on the Routing page as routing, and gives way to the request's own route
- once one is begun (`noteRoute`, `trace.drop`). After `unroutedMost` (2m)
- with no route and nothing written, it is answered 504 in its client's
+ once one is begun (`noteRoute`, `trace.drop`). It is answered 504 after
+ `unroutedUnknown` (30s) held in no wait magpie knows of, or after
+ `unroutedMost` (2m) in all (sharwowwow on X: two minutes of spinning
+ before a 504). The known waits are counted as they happen and their
+ time added up (`noteWait`, `waited`): the request's body arriving
+ (`waitedBody`, every read of `r.Body`), masking (`noteMasking`), System
+ One's or the decision model's classifier (`askWaited`), an image's
+ description (`seenBody`), Codex's auto reset (`autoReset`), plugins'
+ gateway hooks (`GatewayBegin`) and JS gateway middleware, and every
+ `notePhase` (slot, rpm, upstream). What is left is magpie's own work
+ (the catalog, parsing, planning, reading the usage log), ~8.5s at its
+ slowest (a cold catalog sync), so 30s gives it room on a loaded machine.
+ With no route and nothing written, it is answered 504 in its client's
protocol, whole (`Content-Length`, `Connection: close`) so the client
has it while the handler may still be held; its context ends with
`errUnrouted`, its route ends with the error, and the log has every
goroutine's stack (at most once in 10 minutes). The handler's writes go
through `heldWriter`, and nowhere once magpie has answered for it.
- `TestUnroutedRequestShownThenAnswered`, `TestUnroutedShownGivesWayToItsRoute`.
+ `TestUnroutedRequestShownThenAnswered`, `TestUnroutedShownGivesWayToItsRoute`,
+ `TestUnroutedUnknownHoldAnsweredSooner`, `TestUnroutedSlowBodyNotCutOff`,
+ `TestUnroutedMaskingNotCutOff`, `TestUnroutedSlotWaitNotCutOff`.
## Remote magpie APIs
diff --git a/internal/gateway/active.go b/internal/gateway/active.go
index 8eeee6103..4e465ce6c 100644
--- a/internal/gateway/active.go
+++ b/internal/gateway/active.go
@@ -84,6 +84,9 @@ func inflightOf(ctx context.Context) *inflight {
// func that puts back where it stood before: a try out at the vendor whose
// own requests wait for room in the minute (rpmTransport) is out at the
// vendor again once they go.
+//
+// A phase is a wait magpie knows of (noteWait): one before a route is
+// begun has unroutedMost, not unroutedUnknown.
func notePhase(ctx context.Context, phase string) (restore func()) {
f := inflightOf(ctx)
if f == nil {
@@ -91,7 +94,8 @@ func notePhase(ctx context.Context, phase string) (restore func()) {
}
was, _ := f.phase.Load().(string)
f.phase.Store(phase)
- return func() { f.phase.Store(was) }
+ waited := noteWait(ctx)
+ return func() { waited(); f.phase.Store(was) }
}
// noteRoute ties ctx's request in flight to its routing trace, the first
diff --git a/internal/gateway/classify.go b/internal/gateway/classify.go
index 41876540f..8091d1623 100644
--- a/internal/gateway/classify.go
+++ b/internal/gateway/classify.go
@@ -363,6 +363,16 @@ func classifyEffort(model string) string {
// any other model through the gateway itself, as a client would: which of
// intents the message is and, when effort, how hard the turn is, each in a
// call of its own, at once.
+// askWaited is ask with each question a wait ctx's watch knows of
+// (noteWait): a routing group's decision model is asked before the
+// request's route is begun.
+func askWaited(ctx context.Context, ask classifier) classifier {
+ return func(model string, intents []string, prev before, effort bool, text string) (verdict, error) {
+ defer noteWait(ctx)()
+ return ask(model, intents, prev, effort, text)
+ }
+}
+
func (s *Server) askClassifier(model string, intents []string, prev before, effort bool, text string) (verdict, error) {
if p, m, ok := provider.Resolve(model); ok && p.DecidesModel(m) {
return s.askDecider(p, m, intents, prev, effort, text)
diff --git a/internal/gateway/codex_autoreset.go b/internal/gateway/codex_autoreset.go
index 193b95e91..88680b6ba 100644
--- a/internal/gateway/codex_autoreset.go
+++ b/internal/gateway/codex_autoreset.go
@@ -25,6 +25,8 @@ var autoResetTimeout = 20 * time.Second
// c is, so one of them may spend one after cands'. It says which account it
// was and what spending did.
func (s *Server) autoReset(ctx context.Context, cands []candidate, c candidate, held []candidate) (candidate, provider.ResetOutcome, bool) {
+ // spent before a route is begun: a wait magpie knows of (unrouted.go)
+ defer noteWait(ctx)()
for _, x := range cands {
if x.restKey() == c.restKey() {
continue
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go
index 976a21fcb..b33b34f51 100644
--- a/internal/gateway/gateway.go
+++ b/internal/gateway/gateway.go
@@ -2003,6 +2003,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
ask := s.askClassifier
if r.Header.Get("User-Agent") == RouterAgent {
ask = nil
+ } else {
+ // a wait magpie knows of, before a route is begun (unrouted.go)
+ ask = askWaited(r.Context(), ask)
}
if isGroup && slices.ContainsFunc(ms, func(m provider.Member) bool {
return g.Ruled() || slices.ContainsFunc(m.Via, provider.Group.Ruled)
diff --git a/internal/gateway/middleware.go b/internal/gateway/middleware.go
index a8141e1d9..2f12640cb 100644
--- a/internal/gateway/middleware.go
+++ b/internal/gateway/middleware.go
@@ -38,8 +38,12 @@ func (s *Server) serveAgent(w http.ResponseWriter, r *http.Request, from provide
info := gwmw.Info{Protocol: string(from), Model: modelOf(body), Stream: streamOf(body), Path: r.URL.Path, Agent: agentOf(r)}
var tried []gwmw.Tried
for _, a := range agents {
+ // the user's middleware runs before a route is begun: a wait
+ // magpie knows of (unrouted.go)
+ waited := noteWait(r.Context())
run := a.Begin(info)
if run == nil {
+ waited()
continue
}
defer run.End()
@@ -48,6 +52,7 @@ func (s *Server) serveAgent(w http.ResponseWriter, r *http.Request, from provide
}
var turned *gwmw.Reject
body, turned = run.Request(body)
+ waited()
if turned != nil {
writeError(w, from, turned.Status, turned.Message)
return
@@ -98,7 +103,11 @@ func gatewayHooks(w http.ResponseWriter, r *http.Request, from provider.Protocol
info.Cwd = workDir(req)
info.Query = gatewayQuery(req)
}
+ // the plugin host, started for it if need be: a wait magpie knows of
+ // before a route is begun (unrouted.go)
+ waited := noteWait(r.Context())
run := plugin.GatewayBegin(r.Context(), body, info)
+ waited()
if run == nil {
return w, body, func() {}
}
diff --git a/internal/gateway/unrouted.go b/internal/gateway/unrouted.go
index 8102ed30a..59f4e518b 100644
--- a/internal/gateway/unrouted.go
+++ b/internal/gateway/unrouted.go
@@ -5,6 +5,7 @@ import (
"context"
"errors"
"fmt"
+ "io"
"log"
"net/http"
"net/http/httptest"
@@ -33,7 +34,20 @@ import (
// and is answered with an error once it has waited unroutedMost, with
// magpie's log saying where every goroutine stood. Its context ends then
// too, so what waits on it lets go.
-var unroutedShow, unroutedMost = 15 * time.Second, 2 * time.Minute
+//
+// Two minutes of a spinner before that error was a torment of its own
+// (sharwowwow on X). Most of what comes before a route is a wait magpie
+// knows of and that may rightly be long: the body still arriving (an
+// image over a slow link), masking (the local model), the routing
+// group's decision model, an image described for a model that can't see,
+// a Codex reset spent, the plugins' gateway hooks and JS middleware, a
+// slot or a minute's room (noteWait, notePhase). The rest is magpie's own
+// work: parsing, the catalog, the plan. A request that has spent
+// unroutedUnknown in none of those waits, added up however they came, is
+// answered then with the same error; one in those waits still has
+// unroutedMost. 30s is three times the slowest such work a report has
+// measured (a cold catalog sync, ~8.5s) and leaves a loaded machine room.
+var unroutedShow, unroutedUnknown, unroutedMost = 15 * time.Second, 30 * time.Second, 2 * time.Minute
// unroutedWatched is whether a request to path is an agent's turn, which
// is given a route once magpie knows where it goes: Chat, Responses,
@@ -178,6 +192,16 @@ type unroutedHold struct {
// masking counts the maskings under way for it: one held there waits
// for Settings → Privacy's local model, and its error says so
masking atomic.Int32
+
+ // the waits magpie knows it is in (noteWait), under wmu: how many are
+ // under way, since when one has been, the time spent in one before
+ // that, and whether unknown fired in one and looks again at its end
+ wmu sync.Mutex
+ waits int
+ since time.Time
+ excused time.Duration
+ due bool
+ unknown *time.Timer
}
type unroutedKey struct{}
@@ -191,14 +215,107 @@ func noteAsked(ctx context.Context, model string) {
}
// noteMasking tells ctx's watch its request is being masked, until the
-// func it answers is called.
+// func it answers is called: a wait it knows of.
func noteMasking(ctx context.Context) func() {
u, _ := ctx.Value(unroutedKey{}).(*unroutedHold)
if u == nil {
return func() {}
}
u.masking.Add(1)
- return func() { u.masking.Add(-1) }
+ waited := u.wait()
+ return func() { waited(); u.masking.Add(-1) }
+}
+
+// noteWait tells ctx's watch its request is in a wait magpie knows of
+// until the func it answers is called: the time it spends there counts
+// toward unroutedMost alone, not unroutedUnknown.
+func noteWait(ctx context.Context) (done func()) {
+ u, _ := ctx.Value(unroutedKey{}).(*unroutedHold)
+ if u == nil {
+ return func() {}
+ }
+ return u.wait()
+}
+
+// wait begins a known wait, and answers the func that ends it, once.
+func (u *unroutedHold) wait() func() {
+ u.wmu.Lock()
+ if u.waits == 0 {
+ u.since = time.Now()
+ }
+ u.waits++
+ u.wmu.Unlock()
+ var once sync.Once
+ return func() { once.Do(u.waited) }
+}
+
+// waited ends a known wait: the last to end adds its time to excused,
+// and if unknown fired meanwhile it is set for what is left of it.
+func (u *unroutedHold) waited() {
+ u.wmu.Lock()
+ defer u.wmu.Unlock()
+ if u.waits--; u.waits > 0 {
+ return
+ }
+ now := time.Now()
+ u.excused += now.Sub(u.since)
+ if u.due {
+ u.due = false
+ u.unknown.Reset(max(unroutedUnknown-u.unknownLocked(now), 0))
+ }
+}
+
+// unknownLocked is how long the request has been held in no known wait;
+// wmu is held.
+func (u *unroutedHold) unknownLocked(now time.Time) time.Duration {
+ d := now.Sub(u.f.start) - u.excused
+ if u.waits > 0 {
+ d -= now.Sub(u.since)
+ }
+ return d
+}
+
+// excusedFor is how long the request has been held in a known wait.
+func (u *unroutedHold) excusedFor() time.Duration {
+ u.wmu.Lock()
+ defer u.wmu.Unlock()
+ d := u.excused
+ if u.waits > 0 {
+ d += time.Since(u.since)
+ }
+ return d
+}
+
+// checkUnknown is unknown's timer: a request held unroutedUnknown in no
+// known wait is answered for; one in a wait now is looked at again when
+// it ends, and one with time left, when that is up.
+func (u *unroutedHold) checkUnknown() {
+ u.wmu.Lock()
+ if u.waits > 0 {
+ u.due = true
+ u.wmu.Unlock()
+ return
+ }
+ if left := unroutedUnknown - u.unknownLocked(time.Now()); left > 0 {
+ u.unknown.Reset(left)
+ u.wmu.Unlock()
+ return
+ }
+ u.wmu.Unlock()
+ u.giveUp(true)
+}
+
+// waitedBody is a watched request's body: the time spent waiting for it
+// to arrive is a known wait, however much of it is read where.
+type waitedBody struct {
+ io.ReadCloser
+ u *unroutedHold
+}
+
+func (b waitedBody) Read(p []byte) (int, error) {
+ done := b.u.wait()
+ defer done()
+ return b.ReadCloser.Read(p)
}
// watchUnrouted watches r, counted as f, until the func it answers is
@@ -210,15 +327,24 @@ func (s *Server) watchUnrouted(w http.ResponseWriter, r *http.Request, f *inflig
ctx, cancel := context.WithCancelCause(r.Context())
u := &unroutedHold{s: s, f: f, w: newHeldWriter(w), cancel: cancel}
r = r.WithContext(context.WithValue(ctx, unroutedKey{}, u))
+ if r.Body != nil && r.Body != http.NoBody {
+ r.Body = waitedBody{r.Body, u}
+ }
u.r = r
f.mu.Lock()
f.unrouted = u
f.mu.Unlock()
+ u.wmu.Lock()
+ u.unknown = time.AfterFunc(unroutedUnknown, u.checkUnknown)
+ u.wmu.Unlock()
show := time.AfterFunc(unroutedShow, u.show)
- most := time.AfterFunc(unroutedMost, u.giveUp)
+ most := time.AfterFunc(unroutedMost, func() { u.giveUp(false) })
return u.w, r, func() {
show.Stop()
most.Stop()
+ u.wmu.Lock()
+ u.unknown.Stop()
+ u.wmu.Unlock()
f.mu.Lock()
// a timer that fired as the handler returned does nothing now
u.over = true
@@ -270,8 +396,9 @@ func (u *unroutedHold) routed() (drop *Route) {
// giveUp answers the request with an error, when no route was begun for it
// and its handler wrote nothing, ends its context, and says on its route
-// and in the log what held it.
-func (u *unroutedHold) giveUp() {
+// and in the log what held it. unknown: it was held unroutedUnknown in no
+// wait magpie knows of, rather than unroutedMost in all.
+func (u *unroutedHold) giveUp(unknown bool) {
f := u.f
f.mu.Lock()
if len(f.begun) > 0 || u.ended || u.over {
@@ -280,7 +407,10 @@ func (u *unroutedHold) giveUp() {
}
held := time.Since(f.start).Round(time.Second)
msg, where := unroutedMsg(held, u.masking.Load() > 0), ""
- if u.masking.Load() > 0 {
+ switch {
+ case unknown:
+ where = fmt.Sprintf(" %s of it in no wait magpie knows of", unroutedUnknown)
+ case u.masking.Load() > 0:
where = " masking it"
}
if !u.w.answer(unroutedProto(u.r.URL.Path), http.StatusGatewayTimeout, msg) {
diff --git a/internal/gateway/unrouted_test.go b/internal/gateway/unrouted_test.go
index 418400c28..ad32c6bf2 100644
--- a/internal/gateway/unrouted_test.go
+++ b/internal/gateway/unrouted_test.go
@@ -8,8 +8,12 @@ import (
"net/http"
"net/http/httptest"
"strings"
+ "sync"
"testing"
"time"
+
+ "github.com/yetone/magpie/internal/gwmw"
+ "github.com/yetone/magpie/internal/settings"
)
// unroutedLimits sets when a request held before routing shows, and when
@@ -21,6 +25,250 @@ func unroutedLimits(t *testing.T, show, most time.Duration) {
t.Cleanup(func() { unroutedShow, unroutedMost = s, m })
}
+// unknownLimit sets how long a request may be held before routing in no
+// wait magpie knows of.
+func unknownLimit(t *testing.T, d time.Duration) {
+ t.Helper()
+ was := unroutedUnknown
+ unroutedUnknown = d
+ t.Cleanup(func() { unroutedUnknown = was })
+}
+
+// watchedHold is the watch of s's one request held before routing, once
+// it is there.
+func watchedHold(t *testing.T, s *Server) *unroutedHold {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ s.active.mu.Lock()
+ var u *unroutedHold
+ for _, f := range s.active.m {
+ f.mu.Lock()
+ if f.unrouted != nil {
+ u = f.unrouted
+ }
+ f.mu.Unlock()
+ }
+ s.active.mu.Unlock()
+ if u != nil {
+ return u
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("no request was held before routing")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+}
+
+// unroutedReply is a reply postWhole got.
+type unroutedReply struct {
+ code int
+ body string
+ err error
+}
+
+// heldPast waits until u's request has been held in known waits for d,
+// and answers nil then; or answers its reply, if it is answered first.
+func heldPast(t *testing.T, u *unroutedHold, d time.Duration, got chan unroutedReply) *unroutedReply {
+ t.Helper()
+ deadline := time.Now().Add(20 * time.Second)
+ for u.excusedFor() < d {
+ select {
+ case r := <-got:
+ return &r
+ case <-time.After(5 * time.Millisecond):
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("held in known waits for %s only", u.excusedFor())
+ }
+ }
+ return nil
+}
+
+// posted posts body to srv in the background, its reply to the channel.
+func posted(srv *httptest.Server, body io.Reader) chan unroutedReply {
+ got := make(chan unroutedReply, 1)
+ go func() {
+ code, b, err := postWhole(srv, body)
+ got <- unroutedReply{code, b, err}
+ }()
+ return got
+}
+
+// postWhole posts body to srv's Chat API and answers its reply.
+func postWhole(srv *httptest.Server, body io.Reader) (int, string, error) {
+ c := &http.Client{Timeout: 30 * time.Second}
+ resp, err := c.Post(srv.URL+"/v1/chat/completions", "application/json", body)
+ if err != nil {
+ return 0, "", err
+ }
+ b, err := io.ReadAll(resp.Body)
+ resp.Body.Close()
+ return resp.StatusCode, string(b), err
+}
+
+const unroutedHi = `{"model":"probe/text","messages":[{"role":"user","content":"hi"}]}`
+
+// sharwowwow on X: two minutes of a spinner before the 504. A request
+// held before routing in no wait magpie knows of (here a lock routing
+// needs, held and never let go) is answered once it has been held
+// unroutedUnknown, long before unroutedMost.
+func TestUnroutedUnknownHoldAnsweredSooner(t *testing.T) {
+ unroutedLimits(t, 20*time.Millisecond, time.Hour)
+ unknownLimit(t, 300*time.Millisecond)
+ s, _ := slowEyed(t, 0)
+ srv := httptest.NewServer(s.Handler())
+ t.Cleanup(srv.Close)
+ sticks.Lock()
+ var once sync.Once
+ unlock := func() { once.Do(sticks.Unlock) }
+ t.Cleanup(unlock)
+
+ got := posted(srv, strings.NewReader(unroutedHi))
+ var r unroutedReply
+ select {
+ case r = <-got:
+ case <-time.After(20 * time.Second):
+ unlock()
+ t.Fatal("a request held in no known wait was not answered before unroutedMost")
+ }
+ unlock()
+ if r.err != nil || r.code != http.StatusGatewayTimeout || !strings.Contains(r.body, "without routing it to any provider") {
+ t.Fatalf("held request answered %d %q %v", r.code, r.body, r.err)
+ }
+}
+
+// A body still arriving (an image over a slow link) is a known wait: a
+// request whose body takes longer than unroutedUnknown is routed and
+// answered, not cut off.
+func TestUnroutedSlowBodyNotCutOff(t *testing.T) {
+ unroutedLimits(t, 20*time.Millisecond, time.Hour)
+ unknownLimit(t, 100*time.Millisecond)
+ s, _ := slowEyed(t, 0)
+ srv := httptest.NewServer(s.Handler())
+ t.Cleanup(srv.Close)
+ pr, pw := io.Pipe()
+ t.Cleanup(func() { pw.Close() })
+ got := posted(srv, pr)
+ half := len(unroutedHi) / 2
+ if _, err := pw.Write([]byte(unroutedHi[:half])); err != nil {
+ t.Fatal(err)
+ }
+ if r := heldPast(t, watchedHold(t, s), 3*unroutedUnknown, got); r != nil {
+ t.Fatalf("answered %d %q %v while its body was still arriving", r.code, r.body, r.err)
+ }
+ if _, err := pw.Write([]byte(unroutedHi[half:])); err != nil {
+ t.Fatal(err)
+ }
+ pw.Close()
+ r := <-got
+ if r.err != nil || r.code != http.StatusOK {
+ t.Fatalf("a slow body was answered %d %q %v", r.code, r.body, r.err)
+ }
+}
+
+// gatedMasker is masking that waits for the local model until gate is
+// closed, then sends the body as it came.
+type gatedMasker struct {
+ waitingMasker
+ gate chan struct{}
+}
+
+func (m gatedMasker) JSONHitsContext(ctx context.Context, b []byte) ([]byte, []gwmw.Hit) {
+ select {
+ case <-m.gate:
+ case <-ctx.Done():
+ }
+ return b, nil
+}
+
+func (m gatedMasker) TextHitsContext(ctx context.Context, s string) (string, []gwmw.Hit) {
+ return s, nil
+}
+
+func (m gatedMasker) JSONHits(b []byte) ([]byte, []gwmw.Hit) {
+ return m.JSONHitsContext(context.Background(), b)
+}
+func (m gatedMasker) TextHits(s string) (string, []gwmw.Hit) { return s, nil }
+func (m gatedMasker) JSON(b []byte) ([]byte, int) { b, _ = m.JSONHits(b); return b, 0 }
+func (m gatedMasker) Text(s string) (string, int) { return s, 0 }
+
+// Masking (the local model loading) is a known wait: a request masked
+// for longer than unroutedUnknown is routed once masked, not cut off.
+func TestUnroutedMaskingNotCutOff(t *testing.T) {
+ unroutedLimits(t, 20*time.Millisecond, time.Hour)
+ unknownLimit(t, 100*time.Millisecond)
+ s, _ := slowEyed(t, 0)
+ if err := settings.Save(settings.Settings{Redact: true}); err != nil {
+ t.Fatal(err)
+ }
+ withoutMasker(t)
+ m := gatedMasker{gate: make(chan struct{})}
+ var once sync.Once
+ open := func() { once.Do(func() { close(m.gate) }) }
+ t.Cleanup(open)
+ t.Cleanup(gwmw.Mask(m))
+ srv := httptest.NewServer(s.Handler())
+ t.Cleanup(srv.Close)
+ got := posted(srv, strings.NewReader(unroutedHi))
+ u := watchedHold(t, s)
+ if r := heldPast(t, u, 3*unroutedUnknown, got); r != nil {
+ t.Fatalf("answered %d %q %v while it was masked", r.code, r.body, r.err)
+ }
+ if u.masking.Load() == 0 {
+ t.Fatal("held past the limit, but not while masking")
+ }
+ open()
+ r := <-got
+ if r.err != nil || r.code != http.StatusOK {
+ t.Fatalf("a request masked past the limit was answered %d %q %v", r.code, r.body, r.err)
+ }
+}
+
+// A request waiting in a key's queue (MaxConcurrency) past
+// unroutedUnknown is not cut off: the queue is a known wait.
+func TestUnroutedSlotWaitNotCutOff(t *testing.T) {
+ unroutedLimits(t, 20*time.Millisecond, time.Hour)
+ unknownLimit(t, 100*time.Millisecond)
+ s, up := slowEyed(t, 0)
+ srv := httptest.NewServer(s.Handler())
+ t.Cleanup(srv.Close)
+ codes := make(chan int, 2)
+ go func() {
+ code, _, _ := postWhole(srv, strings.NewReader(`{"model":"probe/text","messages":[{"role":"user","content":"hold"}]}`))
+ codes <- code
+ }()
+ <-up.held // the one slot is taken
+ go func() {
+ code, _, _ := postWhole(srv, strings.NewReader(unroutedHi))
+ codes <- code
+ }()
+ deadline := time.Now().Add(20 * time.Second)
+ for {
+ queued := false
+ s.active.mu.Lock()
+ for _, f := range s.active.m {
+ if p, _ := f.phase.Load().(string); p == "slot" && time.Since(f.start) > 3*unroutedUnknown {
+ queued = true
+ }
+ }
+ s.active.mu.Unlock()
+ if queued {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the second request never waited in the queue past the limit")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ up.open()
+ for range 2 {
+ if code := <-codes; code != http.StatusOK {
+ t.Fatalf("a request queued past the limit was answered %d", code)
+ }
+ }
+}
+
// heldSight holds pastedImage before it is routed: its image's
// description is in flight and never ends, as a describe stuck on
// something that never lets go would be. What it holds the request on is
diff --git a/internal/gateway/vision.go b/internal/gateway/vision.go
index 5c50ebd24..6790cfc07 100644
--- a/internal/gateway/vision.go
+++ b/internal/gateway/vision.go
@@ -430,6 +430,9 @@ func (s *Server) askVisionWith(ctx context.Context, model, src, system, text str
// the model would be asked about what it isn't given. One in an older turn
// or a tool's result is left out, as textOnlyBody leaves it.
func (s *Server) seenBody(ctx context.Context, proto provider.Protocol, body []byte, model string) ([]byte, error) {
+ // the images described before a route is begun: a wait magpie knows
+ // of (unrouted.go)
+ defer noteWait(ctx)()
var srcs []string
have := map[string]bool{}
walkImages(proto, body, func(im imageAt) (json.RawMessage, error) {