Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a request held too long before routing leaves a report its user can copy

yetone29185e3bbc6c
diff --git a/docs/subsystems/gateway-routing.md b/docs/subsystems/gateway-routing.md
index e8df252d0..2b0941170 100644
--- a/docs/subsystems/gateway-routing.md
+++ b/docs/subsystems/gateway-routing.md
@@ -348,7 +348,10 @@ recorded as a prediction failure (`noPrediction`).
One's or the decision model's classifier (`askWaited`), an image's
description (`seenBody`), Codex's auto reset (`autoReset`), plugins'
gateway hooks (`GatewayBegin`) and JS gateway middleware, and every
- `notePhase` (slot, rpm, upstream, redact). What is left is magpie's own work
+ `notePhase` (slot, rpm, upstream, redact), and the body room's queue
+ (`bodyRoom.take`, at most `bodyRoomWait`, 100s). Each is named (its
+ `kind`), and the time spent in each kind is kept (`waitKind`) for the
+ report below. What is left is magpie's own work
(the catalog, parsing, planning, reading the usage log), ~8.5s at its
slowest (a cold catalog sync), so 30s gives it room on a loaded machine.
With no route and nothing written, it is answered 504 in its client's
@@ -357,6 +360,23 @@ recorded as a prediction failure (`noPrediction`).
`errUnrouted`, its route ends with the error, and the log has every
goroutine's stack (at most once in 10 minutes). The handler's writes go
through `heldWriter`, and nowhere once magpie has answered for it.
+ The desktop app's log goes only to stderr, which its user can't reach
+ (Jason on Discord, a Codex request held 2m0s, told to send "magpie's
+ log"), so what held it is also saved as a report
+ ([`held_report.go`](../../internal/gateway/held_report.go)): taken while
+ the request is still held, before its context ends, in `HeldDir`
+ (`<cache>/held`, mode 0700), written through a temp file and renamed,
+ the newest `heldKept` (5) kept. It has the time, version and system,
+ method and path, the model and agent, how long it was held, each kind
+ of known wait and its time ("(under way)" for the one it was in), the
+ body room's state, the requests in flight and every goroutine's stack
+ (function names and lines, no values; 128 KB at most, one dump for a
+ burst within 10s). Never the body or headers. Its file name is
+ `Route.Held`; the Routing page's story of that request has Copy Report
+ (`GET /api/gateway/held/{name}`, only a report's own name is read) and,
+ in the app, Open Folder (`POST /api/gateway/held/reveal`). The 504 and
+ the log line say where it is. `TestUnroutedHeldReportSaved`,
+ `TestHeldReportsKeptAndNamed`, `routing-held-report.test.cjs`.
`TestUnroutedRequestShownThenAnswered`, `TestUnroutedShownGivesWayToItsRoute`,
`TestUnroutedUnknownHoldAnsweredSooner`, `TestUnroutedSlowBodyNotCutOff`,
`TestUnroutedMaskingNotCutOff`, `TestUnroutedSlotWaitNotCutOff`,
diff --git a/internal/gateway/active.go b/internal/gateway/active.go
index 23ff75635..a5dfc5241 100644
--- a/internal/gateway/active.go
+++ b/internal/gateway/active.go
@@ -100,7 +100,12 @@ func notePhase(ctx context.Context, phase string) (restore func()) {
}
was, _ := f.phase.Load().(string)
f.phase.Store(phase)
- waited := noteWait(ctx)
+ // masking's phase is named by noteMasking's own wait
+ kind := phase
+ if phase == phaseRedact {
+ kind = ""
+ }
+ waited := noteWait(ctx, kind)
return func() { waited(); f.phase.Store(was) }
}
diff --git a/internal/gateway/body_room.go b/internal/gateway/body_room.go
index 897dcb725..ee2eb35a8 100644
--- a/internal/gateway/body_room.go
+++ b/internal/gateway/body_room.go
@@ -124,7 +124,7 @@ func (b *bodyRoomT) take(ctx context.Context, n int64) error {
}
start := time.Now()
log.Printf("gateway: a %d MiB request body waits for room: %d MiB of %d MiB held (%s)", n>>20, held>>20, room>>20, why)
- defer noteWait(ctx)()
+ defer noteWait(ctx, "body room")()
timer := time.NewTimer(bodyRoomWait)
defer timer.Stop()
var err error
diff --git a/internal/gateway/classify.go b/internal/gateway/classify.go
index 8091d1623..95c2e9763 100644
--- a/internal/gateway/classify.go
+++ b/internal/gateway/classify.go
@@ -368,7 +368,7 @@ func classifyEffort(model string) string {
// request's route is begun.
func askWaited(ctx context.Context, ask classifier) classifier {
return func(model string, intents []string, prev before, effort bool, text string) (verdict, error) {
- defer noteWait(ctx)()
+ defer noteWait(ctx, "decision model")()
return ask(model, intents, prev, effort, text)
}
}
diff --git a/internal/gateway/codex_autoreset.go b/internal/gateway/codex_autoreset.go
index 1b05c8055..da5482b61 100644
--- a/internal/gateway/codex_autoreset.go
+++ b/internal/gateway/codex_autoreset.go
@@ -26,7 +26,7 @@ var autoResetTimeout = 20 * time.Second
// was and what spending did.
func (s *Server) autoReset(ctx context.Context, cands []candidate, c candidate, held []candidate) (candidate, provider.ResetOutcome, bool) {
// spent before a route is begun: a wait magpie knows of (unrouted.go)
- defer noteWait(ctx)()
+ defer noteWait(ctx, "Codex reset")()
for _, x := range cands {
if x.restKey() == c.restKey() {
continue
diff --git a/internal/gateway/held_report.go b/internal/gateway/held_report.go
new file mode 100644
index 000000000..1c8f92da5
--- /dev/null
+++ b/internal/gateway/held_report.go
@@ -0,0 +1,190 @@
+package gateway
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "runtime/pprof"
+ "slices"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/yetone/magpie/internal/appdir"
+)
+
+// A request magpie holds too long before routing it is stopped with a 504
+// (giveUp), and what held it is written to a report of its own: the GUI
+// app's log goes only to a stderr its user never sees, so the log alone
+// can't be sent with a bug report. The report is a small file in HeldDir,
+// the last heldKept of them kept, and the Routing page's row for the
+// request copies it (Route.Held).
+//
+// What it holds: when, magpie's version and system, the method and path,
+// the model asked for and the agent, how long it was held, each kind of
+// wait magpie knew it was in and for how long, the body room's state, how
+// many requests were in flight, and where every goroutine stood (function
+// names and lines, no values). Never the request's body or headers.
+
+// heldKept is how many reports are kept, the newest.
+const heldKept = 5
+
+// heldStacksMost bounds the stacks in one report.
+const heldStacksMost = 128 << 10
+
+// HeldDir is the folder the reports of held requests are saved in.
+func HeldDir() string { return filepath.Join(appdir.Cache(), "held") }
+
+// heldWhere is HeldDir as the error says it: home as ~.
+func heldWhere() string {
+ p := HeldDir()
+ if home, err := os.UserHomeDir(); err == nil && home != "" && strings.HasPrefix(p, home+string(filepath.Separator)) {
+ return "~" + p[len(home):]
+ }
+ return p
+}
+
+// heldName is a report's file name: held-<UTC time>-<nanoseconds>.txt, so
+// names sort in the order written.
+var heldName = regexp.MustCompile(`^held-\d{8}-\d{6}-\d{9}\.txt$`)
+
+// heldMu keeps two reports' writes and prunes apart, and heldLast is
+// when the last was named.
+var (
+ heldMu sync.Mutex
+ heldLast time.Time
+)
+
+// writeHeld saves report in HeldDir, through a temp file renamed into
+// place, keeps the newest heldKept, and answers its file name.
+func writeHeld(report string) (string, error) {
+ heldMu.Lock()
+ defer heldMu.Unlock()
+ dir := HeldDir()
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ return "", err
+ }
+ // a clock coarser than a nanosecond (Windows') names two reports
+ // alike: the later one is a nanosecond on
+ now := time.Now().UTC().Round(0)
+ if !now.After(heldLast) {
+ now = heldLast.Add(time.Nanosecond)
+ }
+ heldLast = now
+ name := fmt.Sprintf("held-%s-%09d.txt", now.Format("20060102-150405"), now.Nanosecond())
+ tmp, err := os.CreateTemp(dir, ".held-*")
+ if err != nil {
+ return "", err
+ }
+ _, err = tmp.WriteString(report)
+ if cerr := tmp.Close(); err == nil {
+ err = cerr
+ }
+ if err == nil {
+ err = os.Rename(tmp.Name(), filepath.Join(dir, name))
+ }
+ if err != nil {
+ os.Remove(tmp.Name())
+ return "", err
+ }
+ if es, err := os.ReadDir(dir); err == nil {
+ var names []string
+ for _, e := range es {
+ if heldName.MatchString(e.Name()) {
+ names = append(names, e.Name())
+ }
+ }
+ slices.Sort(names)
+ for len(names) > heldKept {
+ os.Remove(filepath.Join(dir, names[0]))
+ names = names[1:]
+ }
+ }
+ return name, nil
+}
+
+// HeldReport is the saved report named name, for the Routing page; a name
+// that isn't a report's is refused, so nothing else in the folder or out
+// of it is read.
+func HeldReport(name string) (string, error) {
+ if !heldName.MatchString(name) {
+ return "", os.ErrNotExist
+ }
+ b, err := os.ReadFile(filepath.Join(HeldDir(), name))
+ return string(b), err
+}
+
+// heldReport is the report of u's request, held for held and stopped.
+func (s *Server) heldReport(u *unroutedHold, held time.Duration, who, asked, where string) string {
+ var b strings.Builder
+ fmt.Fprintf(&b, "magpie held a request before routing it, and stopped it with 504\n\n")
+ fmt.Fprintf(&b, "time: %s\n", time.Now().UTC().Format(time.RFC3339))
+ fmt.Fprintf(&b, "magpie: %s (%s/%s, served by %s)\n", cmpOr(Version, "dev"), runtime.GOOS, runtime.GOARCH, cmpOr(Serves, "?"))
+ fmt.Fprintf(&b, "request: %s %s\n", u.r.Method, u.r.URL.Path)
+ fmt.Fprintf(&b, "model: %s\n", cmpOr(asked, "? (not read yet)"))
+ fmt.Fprintf(&b, "agent: %s\n", who)
+ fmt.Fprintf(&b, "held: %s,%s\n", held, where)
+ fmt.Fprintf(&b, "known waits:%s\n", cmpOr(strings.TrimPrefix(u.waitsSaid(), ";"), " none"))
+ bodyRoom.mu.Lock()
+ fmt.Fprintf(&b, "body room: %d of %d bytes held (%s), %d waiting\n", bodyRoom.held, bodyRoom.room, cmpOr(bodyRoom.why, "unset"), len(bodyRoom.queue))
+ bodyRoom.mu.Unlock()
+ s.active.mu.Lock()
+ n := len(s.active.m)
+ s.active.mu.Unlock()
+ fmt.Fprintf(&b, "requests in flight: %d\n", n)
+ fmt.Fprintf(&b, "\nwhere every goroutine stood:\n%s", heldStacks())
+ return b.String()
+}
+
+// waitsSaid is each kind of known wait the request was in and the time
+// spent there, "; " before it, or "" when it was in none.
+func (u *unroutedHold) waitsSaid() string {
+ u.wmu.Lock()
+ defer u.wmu.Unlock()
+ if len(u.kinds) == 0 {
+ return ""
+ }
+ now := time.Now()
+ parts := make([]string, 0, len(u.kinds))
+ for _, k := range u.kinds {
+ d := k.spent
+ if k.n > 0 {
+ d += now.Sub(k.since)
+ }
+ p := fmt.Sprintf("%s %s", k.kind, d.Round(100*time.Millisecond))
+ if k.n > 0 {
+ p += " (under way)"
+ }
+ parts = append(parts, p)
+ }
+ return "; in " + strings.Join(parts, ", ")
+}
+
+// heldStacksAt and heldStacksLast are the last dump of the goroutines for
+// a report, and when: a burst of held requests takes one dump.
+var (
+ heldStacksMu sync.Mutex
+ heldStacksAt time.Time
+ heldStacksLast string
+)
+
+// heldStacks is every goroutine's stack, alike ones counted once, at most
+// heldStacksMost of it.
+func heldStacks() string {
+ heldStacksMu.Lock()
+ defer heldStacksMu.Unlock()
+ if heldStacksLast != "" && time.Since(heldStacksAt) < 10*time.Second {
+ return heldStacksLast
+ }
+ var b bytes.Buffer
+ _ = pprof.Lookup("goroutine").WriteTo(&b, 1)
+ if b.Len() > heldStacksMost {
+ b.Truncate(heldStacksMost)
+ b.WriteString("\n… (cut)\n")
+ }
+ heldStacksAt, heldStacksLast = time.Now(), b.String()
+ return heldStacksLast
+}
diff --git a/internal/gateway/held_report_test.go b/internal/gateway/held_report_test.go
new file mode 100644
index 000000000..50770550d
--- /dev/null
+++ b/internal/gateway/held_report_test.go
@@ -0,0 +1,108 @@
+package gateway
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+// A request held too long before routing leaves a report its user can
+// reach: the GUI app's log goes only to stderr. The report names the wait
+// it was held in, says where every goroutine stood, holds nothing of the
+// request's content, and its route names it for the Routing page; the
+// 504 sends its user to that report, not to a log.
+func TestUnroutedHeldReportSaved(t *testing.T) {
+ t.Setenv("XDG_CACHE_HOME", t.TempDir())
+ unroutedLimits(t, 50*time.Millisecond, time.Second)
+ s, _ := slowEyed(t, 0)
+ heldSight(t, s)
+ srv := httptest.NewServer(s.Handler())
+ t.Cleanup(srv.Close)
+
+ const secret = "hunter2-held-report-secret"
+ body := strings.Replace(pastedImage, `"text":"read"`, `"text":"`+secret+`"`, 1)
+ c := &http.Client{Timeout: 20 * time.Second}
+ resp, err := c.Post(srv.URL+"/v1/chat/completions", "application/json", strings.NewReader(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, _ := io.ReadAll(resp.Body)
+ resp.Body.Close()
+ if resp.StatusCode != http.StatusGatewayTimeout {
+ t.Fatalf("held request answered %d %s", resp.StatusCode, b)
+ }
+ if !strings.Contains(string(b), "Copy Report") || !strings.Contains(string(b), "Routing page") || strings.Contains(string(b), "magpie's log") {
+ t.Fatalf("the 504 doesn't send its user to the report: %s", b)
+ }
+
+ // the 504 goes first, and the report is saved as it is answered
+ var rs []Route
+ for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(10 * time.Millisecond) {
+ rs = routesOf(s, "probe/text")
+ if len(rs) == 1 && rs[0].Held != "" {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("the held request's route names no report: %d routes, held %q", len(rs), func() string {
+ if len(rs) > 0 {
+ return rs[0].Held
+ }
+ return ""
+ }())
+ }
+ }
+ text, err := HeldReport(rs[0].Held)
+ if err != nil {
+ t.Fatalf("its report %q: %v", rs[0].Held, err)
+ }
+ for _, want := range []string{"POST /v1/chat/completions", "probe/text", "image description", "(under way)", "body room:", "goroutine profile:"} {
+ if !strings.Contains(text, want) {
+ t.Errorf("the report doesn't say %q:\n%.2000s", want, text)
+ }
+ }
+ if strings.Contains(text, secret) || strings.Contains(text, "aGVsbG8=") {
+ t.Errorf("the report holds the request's content:\n%.2000s", text)
+ }
+ if fi, err := os.Stat(HeldDir()); err != nil || fi.Mode().Perm()&0o077 != 0 && os.PathSeparator == '/' {
+ t.Errorf("the reports' folder: %v %v", fi, err)
+ }
+}
+
+// The newest heldKept reports are kept, and only a report's own name is
+// read.
+func TestHeldReportsKeptAndNamed(t *testing.T) {
+ t.Setenv("XDG_CACHE_HOME", t.TempDir())
+ var names []string
+ for i := range heldKept + 3 {
+ n, err := writeHeld(fmt.Sprintf("report %d", i))
+ if err != nil {
+ t.Fatal(err)
+ }
+ names = append(names, n)
+ }
+ es, err := os.ReadDir(HeldDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(es) != heldKept {
+ t.Fatalf("%d files kept, want %d: %v", len(es), heldKept, es)
+ }
+ if _, err := HeldReport(names[0]); err == nil {
+ t.Error("the oldest report was kept")
+ }
+ if got, err := HeldReport(names[len(names)-1]); err != nil || got != fmt.Sprintf("report %d", heldKept+2) {
+ t.Errorf("the newest report: %q %v", got, err)
+ }
+ os.WriteFile(filepath.Join(HeldDir(), "other.txt"), []byte("x"), 0o600)
+ for _, bad := range []string{"other.txt", "../held/" + names[len(names)-1], "", ".held-1"} {
+ if _, err := HeldReport(bad); err == nil {
+ t.Errorf("HeldReport(%q) read a file that isn't a report", bad)
+ }
+ }
+}
diff --git a/internal/gateway/middleware.go b/internal/gateway/middleware.go
index c7978d87d..c46ac21f6 100644
--- a/internal/gateway/middleware.go
+++ b/internal/gateway/middleware.go
@@ -40,7 +40,7 @@ func (s *Server) serveAgent(w http.ResponseWriter, r *http.Request, from provide
for _, a := range agents {
// the user's middleware runs before a route is begun: a wait
// magpie knows of (unrouted.go)
- waited := noteWait(r.Context())
+ waited := noteWait(r.Context(), "JS middleware")
run := a.Begin(info)
if run == nil {
waited()
@@ -119,7 +119,7 @@ func gatewayHooks(w http.ResponseWriter, r *http.Request, from provider.Protocol
}
// the plugin host, started for it if need be: a wait magpie knows of
// before a route is begun (unrouted.go)
- waited := noteWait(r.Context())
+ waited := noteWait(r.Context(), "plugin hooks")
run := plugin.GatewayBegin(r.Context(), body, info)
waited()
if run == nil {
diff --git a/internal/gateway/trace.go b/internal/gateway/trace.go
index ae11d9aa7..9e06af563 100644
--- a/internal/gateway/trace.go
+++ b/internal/gateway/trace.go
@@ -125,6 +125,9 @@ type Route struct {
// Masking: the ms of Own that went on masking it (maskingMs)
Phase string `json:"phase,omitempty"`
Masking int64 `json:"masking,omitempty"`
+ // Held: the file name, in HeldDir, of the report saved of a request
+ // magpie held too long before routing it and stopped (giveUp)
+ Held string `json:"held,omitempty"`
// Served: the model the reply says answered, as the last try has it;
// Swapped: another than the one that try asked for; Routed: that try
// asked another magpie's routing group, and Served is its member
diff --git a/internal/gateway/unrouted.go b/internal/gateway/unrouted.go
index 30da52386..921aef9d4 100644
--- a/internal/gateway/unrouted.go
+++ b/internal/gateway/unrouted.go
@@ -9,6 +9,7 @@ import (
"log"
"net/http"
"net/http/httptest"
+ "path/filepath"
"runtime/pprof"
"strconv"
"strings"
@@ -216,6 +217,9 @@ type unroutedHold struct {
excused time.Duration
due bool
unknown *time.Timer
+ // each kind of known wait it was in, in the order first met, for the
+ // report of where it was held (heldReport)
+ kinds []*waitKind
// the same for the maskings under way (noteMasking), whose time
// counts toward unroutedMost no more: how many, since when, the time
@@ -246,7 +250,7 @@ func noteMasking(ctx context.Context) func() {
return func() {}
}
u.masking.Add(1)
- waited := u.wait()
+ waited := u.wait("masking")
u.wmu.Lock()
if u.mwaits == 0 {
u.msince = time.Now()
@@ -311,24 +315,64 @@ func (u *unroutedHold) checkMost() {
// noteWait tells ctx's watch its request is in a wait magpie knows of
// until the func it answers is called: the time it spends there counts
// toward unroutedMost alone, not unroutedUnknown.
-func noteWait(ctx context.Context) (done func()) {
+// kind names the wait in the report of a request held too long
+// (heldReport): "body", "body room", "hooks" and the rest.
+func noteWait(ctx context.Context, kind string) (done func()) {
u, _ := ctx.Value(unroutedKey{}).(*unroutedHold)
if u == nil {
return func() {}
}
- return u.wait()
+ return u.wait(kind)
}
-// wait begins a known wait, and answers the func that ends it, once.
-func (u *unroutedHold) wait() func() {
+// waitKind is the time a request spent in one kind of known wait: how
+// many of it are under way, since when, and the time before that.
+type waitKind struct {
+ kind string
+ n int
+ since time.Time
+ spent time.Duration
+}
+
+// wait begins a known wait of kind, and answers the func that ends it,
+// once. A kind of "" is counted but not named: a phase whose time its own
+// wait names already.
+func (u *unroutedHold) wait(kind string) func() {
u.wmu.Lock()
+ now := time.Now()
if u.waits == 0 {
- u.since = time.Now()
+ u.since = now
}
u.waits++
+ var k *waitKind
+ if kind != "" {
+ for _, x := range u.kinds {
+ if x.kind == kind {
+ k = x
+ }
+ }
+ if k == nil {
+ k = &waitKind{kind: kind}
+ u.kinds = append(u.kinds, k)
+ }
+ if k.n++; k.n == 1 {
+ k.since = now
+ }
+ }
u.wmu.Unlock()
var once sync.Once
- return func() { once.Do(u.waited) }
+ return func() {
+ once.Do(func() {
+ if k != nil {
+ u.wmu.Lock()
+ if k.n--; k.n == 0 {
+ k.spent += time.Since(k.since)
+ }
+ u.wmu.Unlock()
+ }
+ u.waited()
+ })
+ }
}
// waited ends a known wait: the last to end adds its time to excused,
@@ -395,7 +439,7 @@ type waitedBody struct {
}
func (b waitedBody) Read(p []byte) (int, error) {
- done := b.u.wait()
+ done := b.u.wait("body")
defer done()
return b.ReadCloser.Read(p)
}
@@ -511,9 +555,22 @@ func (u *unroutedHold) giveUp(unknown bool) {
u.s.trace.update(shown, func(r *Route) {
r.Done, r.Status, r.Error, r.Millis = true, http.StatusGatewayTimeout, msg, time.Since(r.Time).Milliseconds()
})
- u.cancel(errUnrouted)
+ // the report is taken while it is still held: once its context ends,
+ // what held it lets go, and the stacks would show it gone
asked, _ := u.asked.Load().(string)
- log.Printf("gateway: %s %s for %s from %s held %s before routing,%s; stopped with 504%s", u.r.Method, u.r.URL.Path, cmpOr(asked, "?"), cmpOr(f.agent, "?"), held, cmpOr(where, " not masking it"), unroutedStacks())
+ who := cmpOr(u.agent, cmpOr(f.agent, "?"))
+ where = cmpOr(where, " not masking it")
+ report := u.s.heldReport(u, held, who, asked, where)
+ u.cancel(errUnrouted)
+ name, err := writeHeld(report)
+ saved := ""
+ if err != nil {
+ saved = "; its report could not be saved: " + err.Error()
+ } else {
+ saved = "; its report is " + filepath.Join(HeldDir(), name)
+ u.s.trace.update(shown, func(r *Route) { r.Held = name })
+ }
+ log.Printf("gateway: %s %s for %s from %s held %s before routing,%s%s; stopped with 504%s%s", u.r.Method, u.r.URL.Path, cmpOr(asked, "?"), who, held, where, u.waitsSaid(), saved, unroutedStacks())
}
// unroutedMsg is the error of a request held for held before routing:
@@ -530,11 +587,16 @@ func unroutedMsg(held time.Duration, masking bool) string {
return fmt.Sprintf("magpie held this request for %s waiting for the local masking model (Settings → Privacy) to mask it%s, so it was stopped rather than sent masked by the rules alone. Send it again. To send requests without waiting for the model, turn it off in Settings → Privacy.", held, why)
}
if masking {
- return fmt.Sprintf("magpie held this request for %s while masking it (Settings → Privacy), before routing it to any provider, so it was stopped rather than left waiting. Send it again; if it keeps happening, restart magpie and report it with magpie's log, which says where it was held.", held)
+ return fmt.Sprintf("magpie held this request for %s while masking it (Settings → Privacy), before routing it to any provider, so it was stopped rather than left waiting. Send it again. "+heldSay, held, heldWhere())
}
- return fmt.Sprintf("magpie held this request for %s without routing it to any provider, so it was stopped rather than left waiting. Send it again; if it keeps happening, restart magpie and report it with magpie's log, which says where it was held.", held)
+ return fmt.Sprintf("magpie held this request for %s without routing it to any provider, so it was stopped rather than left waiting. Send it again. "+heldSay, held, heldWhere())
}
+// heldSay is where the error of a request held too long sends its user:
+// the report magpie saved of it (heldReport), which a desktop app's user
+// can reach, where its log goes only to a stderr nobody sees.
+const heldSay = "If it keeps happening, report it with the report magpie saved of where it was held: Copy Report on this request on magpie's Routing page, or the newest file in %s."
+
// modelOn and modelBusy are internal/masking's, a test's
// to stand in for.
var modelOn, modelBusy = masking.ModelOn, masking.ModelBusy
diff --git a/internal/gateway/vision.go b/internal/gateway/vision.go
index f44917618..dc2dfd857 100644
--- a/internal/gateway/vision.go
+++ b/internal/gateway/vision.go
@@ -432,7 +432,7 @@ func (s *Server) askVisionWith(ctx context.Context, model, src, system, text str
func (s *Server) seenBody(ctx context.Context, proto provider.Protocol, body []byte, model string) ([]byte, error) {
// the images described before a route is begun: a wait magpie knows
// of (unrouted.go)
- defer noteWait(ctx)()
+ defer noteWait(ctx, "image description")()
var srcs []string
have := map[string]bool{}
walkImages(proto, body, func(im imageAt) (json.RawMessage, error) {
diff --git a/internal/gui/api.go b/internal/gui/api.go
index e0953b006..85efcd9e9 100644
--- a/internal/gui/api.go
+++ b/internal/gui/api.go
@@ -2407,6 +2407,20 @@ func Handler(w Windows, gw *gateway.Server) http.Handler {
}
rw.WriteHeader(http.StatusNoContent)
})
+ // the folder of the reports saved of requests held too long before
+ // routing (gateway.HeldDir), from such a request's row on Routing
+ mux.HandleFunc("POST /api/gateway/held/reveal", func(rw http.ResponseWriter, r *http.Request) {
+ dir := gateway.HeldDir()
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ fail(rw, err)
+ return
+ }
+ if err := w.OpenFolder(dir); err != nil {
+ fail(rw, err)
+ return
+ }
+ rw.WriteHeader(http.StatusNoContent)
+ })
mux.HandleFunc("POST /api/window/{action}", func(rw http.ResponseWriter, r *http.Request) {
switch r.PathValue("action") {
case "hide":
diff --git a/internal/gui/assets/i18n.js b/internal/gui/assets/i18n.js
index 74b622952..9bb903222 100644
--- a/internal/gui/assets/i18n.js
+++ b/internal/gui/assets/i18n.js
@@ -4082,6 +4082,8 @@ const I18N = {
"1 place": "1 处",
"as written": "原样",
"Where": "位置",
+ "Copy Report": "复制报告",
+ "This report is no longer kept: magpie keeps the last 5.": "这份报告已经不在了:magpie 只保留最近 5 份。",
},
"zh-TW": {
"MCP hub": "MCP 聚合",
@@ -8147,6 +8149,8 @@ const I18N = {
"1 place": "1 處",
"as written": "原樣",
"Where": "位置",
+ "Copy Report": "複製報告",
+ "This report is no longer kept: magpie keeps the last 5.": "這份報告已經不在了:magpie 只保留最近 5 份。",
},
ja: {
"MCP hub": "MCP ハブ",
@@ -12214,6 +12218,8 @@ const I18N = {
"1 place": "1 か所",
"as written": "そのまま",
"Where": "場所",
+ "Copy Report": "レポートをコピー",
+ "This report is no longer kept: magpie keeps the last 5.": "このレポートはもう残っていません。magpie は最新の 5 件だけを保存します。",
},
de: {
"MCP hub": "MCP-Hub",
@@ -16281,6 +16287,8 @@ const I18N = {
"1 place": "1 Stelle",
"as written": "unverändert",
"Where": "Wo",
+ "Copy Report": "Bericht kopieren",
+ "This report is no longer kept: magpie keeps the last 5.": "Dieser Bericht ist nicht mehr da: magpie behält nur die letzten 5.",
},
};
diff --git a/internal/gui/assets/plugins/routing/client.js b/internal/gui/assets/plugins/routing/client.js
index cb99a9c0f..af2a5c47c 100644
--- a/internal/gui/assets/plugins/routing/client.js
+++ b/internal/gui/assets/plugins/routing/client.js
@@ -2855,6 +2855,11 @@ function routingPage(part) {
if (tr.done && tryOk(tr) && tr.upstream) items.push([upstreamWhy(tr), "aside upstream-said", tr]);
});
if (r.done && !r.tries.length) items.push([t("Nothing was tried: {error}", { error: r.error || r.status }), "bad"]);
+ // one magpie held too long before routing it was stopped, and what
+ // held it saved to a report (the app's log goes only to stderr, which
+ // its user can't reach): its line copies it. Its name rides as the
+ // line's model, so another held request's line is drawn anew.
+ if (r.done && r.held) items.push([t("magpie saved a report of where it held this request. If it keeps happening, send the report with yours."), "aside held", { model: r.held }]);
const aboutKey = (w) => w ? seat(w) : "";
const logos = [...r.order, ...(r.left || [])].map(logoOf);
const key = JSON.stringify([tagKind(r), items.map(([s, c, tr, w]) => [s, c, tr?.model, tr?.served, aboutKey(w)]), logos]);
@@ -2886,11 +2891,30 @@ function routingPage(part) {
if (c === "aside kind") li.prepend(kindTag(r), " ");
if (c === "swap") li.prepend(swapTag(tr), " ");
if (c === "aside time") li.replaceChildren(timeElement(tr));
+ if (c === "aside held") li.append(" ", ...heldActions(tr.model));
return li;
}));
keptIcons = null;
}
+ // heldActions are a held request's report's buttons: Copy Report, and
+ // the folder it is kept in, on this computer only
+ function heldActions(name) {
+ const cp = el("button", "link", t("Copy Report"));
+ cp.onclick = async () => {
+ try {
+ const got = await api("gateway/held/" + encodeURIComponent(name));
+ await copy(got.text, "", cp, t("Report copied"));
+ } catch (e) {
+ status(t("Couldn't read the report: {error}", { error: t(e.message) }), "err");
+ }
+ };
+ if (window.bootPrefs?.web) return [cp];
+ const open = el("button", "link", t("Open Folder"));
+ open.onclick = () => api("gateway/held/reveal", {}).catch((e) => status(e.message, "err"));
+ return [cp, " ", open];
+ }
+
// pick sets the stage to a past request, or back to live with the newest.
// The page stays where it is: a request clicked in the list stays under
// the pointer, and the stage and its story change above it (it used to
diff --git a/internal/gui/assets/plugins/routing/i18n.js b/internal/gui/assets/plugins/routing/i18n.js
index 5d3c207d0..f12ffa05c 100644
--- a/internal/gui/assets/plugins/routing/i18n.js
+++ b/internal/gui/assets/plugins/routing/i18n.js
@@ -580,6 +580,10 @@ magpie.strings({
"To ask for the maker's own API, tick “{tick}” in the provider's settings.": "要指定模型厂商的官方 API,请在该服务商设置里勾选“{tick}”。",
"No provider was asked for, and its reply didn't say which one answered.": "请求时没有指定上游,回复也没有说是哪家作答。",
"{h} h {m} min": "{h} 小时 {m} 分钟",
+ "magpie saved a report of where it held this request. If it keeps happening, send the report with yours.": "magpie 保存了一份报告,记下这个请求卡在了哪里。如果反复出现,请把这份报告附在你的反馈里。",
+ "Report copied": "报告已复制",
+ "Couldn't read the report: {error}": "读不到这份报告:{error}",
+ "Open Folder": "打开文件夹",
},
"zh-TW": {
"magpie is masking this request before routing it (Settings › Privacy), so who takes it isn't picked yet.": "magpie 正在為這個請求脫敏(設定 › 隱私),脫敏完才路由,所以還沒選出由誰處理。",
@@ -1160,6 +1164,10 @@ magpie.strings({
"To ask for the maker's own API, tick “{tick}” in the provider's settings.": "要指定模型廠商的官方 API,請在該服務商設定裡勾選「{tick}」。",
"No provider was asked for, and its reply didn't say which one answered.": "請求時沒有指定上游,回覆也沒有說是哪家作答。",
"{h} h {m} min": "{h} 小時 {m} 分鐘",
+ "magpie saved a report of where it held this request. If it keeps happening, send the report with yours.": "magpie 儲存了一份報告,記下這個請求卡在了哪裡。如果反覆出現,請把這份報告附在你的回報裡。",
+ "Report copied": "報告已複製",
+ "Couldn't read the report: {error}": "讀不到這份報告:{error}",
+ "Open Folder": "打開資料夾",
},
ja: {
"magpie is masking this request before routing it (Settings › Privacy), so who takes it isn't picked yet.": "magpie はルーティングの前にこのリクエストをマスクしています(設定 › プライバシー)。誰が処理するかはまだ決まっていません。",
@@ -1740,6 +1748,10 @@ magpie.strings({
"{h} h {m} min": "{h} 時間 {m} 分",
"This routing group will no longer be available to agents.": "このルーティンググループはエージェントから利用できなくなります。",
"These routing groups will no longer be available to agents.": "これらのルーティンググループはエージェントから利用できなくなります。",
+ "magpie saved a report of where it held this request. If it keeps happening, send the report with yours.": "magpie はこのリクエストがどこで止まっていたかをレポートに保存しました。繰り返し起きる場合は、報告にこのレポートを添えてください。",
+ "Report copied": "レポートをコピーしました",
+ "Couldn't read the report: {error}": "レポートを読み込めませんでした:{error}",
+ "Open Folder": "フォルダを開く",
},
de: {
"magpie is masking this request before routing it (Settings › Privacy), so who takes it isn't picked yet.": "magpie maskiert diese Anfrage vor der Weiterleitung (Einstellungen › Datenschutz), daher steht noch nicht fest, wer sie übernimmt.",
@@ -2320,5 +2332,9 @@ magpie.strings({
"{h} h {m} min": "{h} h {m} min",
"This routing group will no longer be available to agents.": "Diese Weiterleitungsgruppe steht Agenten nicht mehr zur Verfügung.",
"These routing groups will no longer be available to agents.": "Diese Weiterleitungsgruppen stehen Agenten nicht mehr zur Verfügung.",
+ "magpie saved a report of where it held this request. If it keeps happening, send the report with yours.": "magpie hat in einem Bericht festgehalten, wo diese Anfrage hängen blieb. Passiert es wieder, senden Sie den Bericht mit Ihrer Meldung mit.",
+ "Report copied": "Bericht kopiert",
+ "Couldn't read the report: {error}": "Der Bericht konnte nicht gelesen werden: {error}",
+ "Open Folder": "Ordner öffnen",
},
});
diff --git a/internal/gui/tests/routing-held-report.test.cjs b/internal/gui/tests/routing-held-report.test.cjs
new file mode 100644
index 000000000..c834e6146
--- /dev/null
+++ b/internal/gui/tests/routing-held-report.test.cjs
@@ -0,0 +1,118 @@
+// Run with Node's test runner and Playwright on the module path; see README.md.
+// Jason on Discord: a Codex turn magpie held 2m0s before routing it was
+// stopped with a 504 that said to report it "with magpie's log", and the
+// desktop app's log goes only to a stderr its user never sees. magpie now
+// saves a report of where it was held, and the request's story on the
+// Routing page copies it (Copy Report) and opens its folder (Open Folder,
+// on this computer only, not in magpie web). A request with no report
+// has no such line. In every language, both engines, wide and narrow.
+const assert = require("node:assert/strict");
+const fs = require("node:fs/promises");
+const path = require("node:path");
+const { test } = require("node:test");
+const { chromium, webkit } = require("playwright");
+
+const assets = path.resolve(__dirname, "../assets");
+const now = new Date();
+const ago = (minutes) => new Date(now.getTime() - minutes * 60e3).toISOString();
+const name = "held-20261029-045944-499212000.txt";
+const report = "magpie held a request before routing it, and stopped it with 504\n\nknown waits: in body room 1m40s (under way)\n";
+
+const route = (held) => ({ id: 1, seq: 1, time: ago(1), agent: "AirbuildEnvironment", model: "gpt-6-astra", provider: "magpie", done: true, status: 504, ms: 120000,
+ error: "magpie held this request for 2m0s without routing it to any provider, so it was stopped rather than left waiting.", order: [], tries: [], ...(held ? { held: name } : {}) });
+
+function serve(lang, { web, held, seen }) {
+ return async (request) => {
+ const url = new URL(request.request().url());
+ const json = (data) => request.fulfill({ json: data });
+ if (url.pathname === "/boot.js") return request.fulfill({ contentType: "text/javascript", body: `window.bootPrefs = {lang:"${lang}",theme:"light",web:${web}};` });
+ if (url.pathname === "/wails/runtime.js") return request.fulfill({ contentType: "text/javascript", body: "export const Window = {};" });
+ if (url.pathname === "/api/state") return json({ agents: [{ id: "codex", name: "Codex", path: "/test/config.toml", fields: [] }], profiles: [], settings: { lang, theme: "light" } });
+ if (url.pathname === "/api/gateway/trace") {
+ if (url.searchParams.get("wait")) return new Promise(() => {}); // nothing more comes
+ return json({ mine: true, now: now.toISOString(), seq: 1, totals: { requests: 0 }, routes: [route(held)], ids: [1] });
+ }
+ if (url.pathname === "/api/gateway/held/" + name) { seen.read++; return json({ text: report }); }
+ if (url.pathname === "/api/gateway/held/reveal") { seen.reveal++; return request.fulfill({ status: 204 }); }
+ if (url.pathname === "/api/copy") { seen.copied.push(JSON.parse(request.request().postData()).text); return json({}); }
+ if (url.pathname === "/api/gateway/history") return json({ cut: false, days: [], routes: [] });
+ if (url.pathname === "/api/groups") return json({ models: [], groups: [], pools: [] });
+ if (url.pathname === "/api/providers") return json({ providers: [], presets: [], excluded: [], gateway: { running: true, window: true } });
+ if (url.pathname.startsWith("/api/")) return json({});
+ const file = path.join(assets, url.pathname === "/" ? "index.html" : url.pathname);
+ const contentType = { ".html": "text/html", ".js": "text/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".png": "image/png" }[path.extname(file)];
+ await request.fulfill({ body: await fs.readFile(file), contentType });
+ };
+}
+
+const said = {
+ en: ["magpie saved a report of where it held this request. If it keeps happening, send the report with yours.", "Copy Report", "Open Folder"],
+ zh: ["magpie 保存了一份报告,记下这个请求卡在了哪里。如果反复出现,请把这份报告附在你的反馈里。", "复制报告", "打开文件夹"],
+ "zh-TW": ["magpie 儲存了一份報告,記下這個請求卡在了哪裡。如果反覆出現,請把這份報告附在你的回報裡。", "複製報告", "打開資料夾"],
+ ja: ["magpie はこのリクエストがどこで止まっていたかをレポートに保存しました。繰り返し起きる場合は、報告にこのレポートを添えてください。", "レポートをコピー", "フォルダを開く"],
+ de: ["magpie hat in einem Bericht festgehalten, wo diese Anfrage hängen blieb. Passiert es wieder, senden Sie den Bericht mit Ihrer Meldung mit.", "Bericht kopieren", "Ordner öffnen"],
+};
+
+for (const engine of (process.env.BROWSER ? [process.env.BROWSER] : ["chromium", "webkit"])) {
+ for (const lang of ["en", "zh", "zh-TW", "ja", "de"]) {
+ for (const width of [1100, 420]) {
+ for (const web of [false, true]) {
+ test(`${engine} ${lang} ${width}px ${web ? "web" : "app"}: a held request's story copies its report`, async (t) => {
+ const browser = await (engine === "webkit" ? webkit.launch() : chromium.launch({ channel: "chromium" }));
+ t.after(() => browser.close());
+ const page = await browser.newPage({ viewport: { width, height: 700 }, reducedMotion: "reduce" });
+ page.setDefaultTimeout(5000);
+ const errors = [];
+ page.on("pageerror", (error) => errors.push(error.message));
+ const seen = { read: 0, reveal: 0, copied: [] };
+ await page.route("**/*", serve(lang, { web, held: true, seen }));
+ await page.goto("http://magpie.test/?view=routing");
+ const [line, copyWord, openWord] = said[lang];
+ const li = page.locator(".rt-steps li.held");
+ await li.waitFor();
+ assert.ok((await li.textContent()).startsWith(line), await li.textContent());
+ const buttons = li.locator("button.link");
+ assert.deepEqual((await buttons.allTextContents()).map((s) => s.trim()), web ? [copyWord] : [copyWord, openWord]);
+ // the buttons are on the page, not cut off at its edge, and the
+ // reader scrolls the page to them: magpie keeps the page where its
+ // reader left it (no unasked scroll), so a click's own scroll is
+ // undone and the page is scrolled by the wheel, as a reader does
+ const view = page.locator("#view-routing");
+ const vb = await view.boundingBox();
+ await page.mouse.move(vb.x + vb.width / 2, vb.y + vb.height / 2);
+ for (let i = 0; i < 20; i++) {
+ const box = await buttons.last().boundingBox();
+ if (box && box.y + box.height <= vb.y + vb.height) break;
+ await page.mouse.wheel(0, 120);
+ await page.waitForTimeout(30);
+ }
+ for (const b of await buttons.all()) {
+ const box = await b.boundingBox();
+ assert.ok(box && box.x >= 0 && box.x + box.width <= width && box.y >= vb.y && box.y + box.height <= vb.y + vb.height, `a button is out of sight: ${JSON.stringify(box)} in ${JSON.stringify(vb)}`);
+ }
+ await buttons.first().click();
+ for (let i = 0; i < 250 && !seen.copied.length; i++) await page.waitForTimeout(20);
+ assert.equal(seen.read, 1, "Copy Report read the report");
+ assert.deepEqual(seen.copied, [report], "Copy Report copied the report itself");
+ if (!web) {
+ await buttons.nth(1).click();
+ for (let i = 0; i < 250 && !seen.reveal; i++) await page.waitForTimeout(20);
+ assert.equal(seen.reveal, 1, "Open Folder asked for the reports' folder");
+ }
+ assert.deepEqual(errors, []);
+ });
+ }
+ }
+ test(`${engine} ${lang}: a held request with no report has no report line`, async (t) => {
+ const browser = await (engine === "webkit" ? webkit.launch() : chromium.launch({ channel: "chromium" }));
+ t.after(() => browser.close());
+ const page = await browser.newPage({ viewport: { width: 1100, height: 700 }, reducedMotion: "reduce" });
+ page.setDefaultTimeout(5000);
+ const seen = { read: 0, reveal: 0, copied: [] };
+ await page.route("**/*", serve(lang, { web: false, held: false, seen }));
+ await page.goto("http://magpie.test/?view=routing");
+ await page.locator(".rt-steps li.bad").first().waitFor();
+ assert.equal(await page.locator(".rt-steps li.held").count(), 0);
+ });
+ }
+}
diff --git a/internal/gui/trace.go b/internal/gui/trace.go
index 6b385f489..1b62afd9e 100644
--- a/internal/gui/trace.go
+++ b/internal/gui/trace.go
@@ -386,6 +386,20 @@ func traceRoutes(mux *http.ServeMux) {
rememberTitleHistory(day, routes)
writeJSON(rw, map[string]any{"days": days, "routes": pricedRoutes(routes), "cut": cut})
})
+ // the report saved of a request held too long before routing, for its
+ // row's Copy Report: the app's log goes only to stderr, which its user
+ // can't reach. A magpie serving the gateway for this one (OwnerTrace)
+ // saves it in the same folder.
+ mux.HandleFunc("GET /api/gateway/held/{name}", func(rw http.ResponseWriter, r *http.Request) {
+ text, err := gateway.HeldReport(r.PathValue("name"))
+ if err != nil {
+ rw.Header().Set("Content-Type", "application/json")
+ rw.WriteHeader(http.StatusNotFound)
+ _ = json.NewEncoder(rw).Encode(map[string]string{"error": "This report is no longer kept: magpie keeps the last 5."})
+ return
+ }
+ writeJSON(rw, map[string]string{"text": text})
+ })
// an account's rest lifted by hand: verified with its vendor, say
mux.HandleFunc("POST /api/gateway/unrest", func(rw http.ResponseWriter, r *http.Request) {
var in struct {