Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a picked model that turns an image away gets it described, not another provider (xyenon)

yetoneb64ec9d64f80
diff --git a/docs/subsystems/gateway-routing.md b/docs/subsystems/gateway-routing.md
index eaf688116..151f5174c 100644
--- a/docs/subsystems/gateway-routing.md
+++ b/docs/subsystems/gateway-routing.md
@@ -906,6 +906,39 @@ Covered by `TestSearchMCPServesWebSearch`, `TestSearchMCPWithNoSearcher`,
`TestSearchMCPGuarded`, `TestSearchMCPHeldToKeyLimit`,
`TestSearchMCPRoutedAsTheAgentsCall`, `TestSearchQueryMasked` and `search-mcp.test.cjs`.
+## A model that turns an image away
+
+Whether a model sees is its provider's list's word (`ImageInput`), else
+the served entry's (`blindTo`), and a list that says nothing takes
+models.dev's modalities for the model (`catalog.SeesImages`). A relay's
+copy of a model can take text only where models.dev says it sees: xyenon
+(Discord) picked provider-1/DeepSeek-V4.1-Flash in pi with Image
+recognition set, magpie sent it the raw image, DeepSeek answered 400
+"unknown variant `image_url`", and since that reads as a shape refusal
+(`shapeWords`) the request went on to the fallback, the same model at
+provider-2, which sees. The image was never described.
+
+A 400, 415 or 422 whose words say images aren't taken (`imageRefused`:
+unknown variant image_url/input_image/image, OpenAI's "only supported by
+certain models", OpenRouter's "no endpoints found that support image",
+vLLM's "not a multimodal model", 不支持图片 and the like) now marks the
+model blind for the server's run (`refuseSight`, `Server.blind`;
+`blindTo` checks it first). With a describer (`seer`), the same candidate
+is asked again at once with the images described (`seenBody`, or the
+group's `seenGroup`; the try's failure is `failSight`), before any
+fallback or other member, and later requests are described up front, for
+a direct pick and a group alike. The pick and the describer the user set
+win over another provider's copy of the model.
+
+With no describer (Image recognition off, or nothing that sees), going on
+to a candidate whose copy sees is the only way to answer, so that is kept
+on purpose (`TestRefusedImageWithNoDescriberGoesToOneThatSees`).
+
+Covered by `TestPickedModelThatRefusesAnImageIsGivenItDescribed` (direct,
+user group and auto group, over Chat, Responses and Messages) and
+`TestImageRefused`
+([`vision_refused_test.go`](../../internal/gateway/vision_refused_test.go)).
+
## Image recognition as an MCP server
`VisionMCPPath`, `/mcp/magpie/vision`, serves the model that describes
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go
index 6797ce8ff..80d8ae86a 100644
--- a/internal/gateway/gateway.go
+++ b/internal/gateway/gateway.go
@@ -466,6 +466,10 @@ type Server struct {
sightMu sync.Mutex
sights map[string]*sight
sightOrder []string
+ // blind are the models that turned an image away while this server
+ // ran, by "<provider>/<model>" (refuseSight): kept for the run alone,
+ // so a vendor that adds sight is asked again after a restart
+ blind sync.Map
// the requests out at each key or account with a MaxConcurrency, and
// those waiting their turn (concurrency.go)
lanes lanes
@@ -2168,11 +2172,16 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
// a model its list says nothing of is text-only to a describer, as
// agents are told: while one describes images they are told every
// model takes them, and send the images on to be described
+ //
+ // One that turned an image away since magpie started is text-only
+ // whatever its list or models.dev says, with a model to describe the
+ // image: the model picked is given the description, not left to fail
+ // over to the same model at another provider that sees (xyenon)
unknownSight := false
- if imageInput == nil && !isGroup && hasImage(from, body) && blindTo(p.ID, model, nil) {
+ if !isGroup && (imageInput == nil || *imageInput) && hasImage(from, body) && s.blindTo(p.ID, model, imageInput) {
if _, ok := seeing(); ok {
no := false
- imageInput, unknownSight = &no, true
+ imageInput, unknownSight = &no, imageInput == nil && !s.sightRefused(p.ID, model)
}
}
if imageInput != nil && !*imageInput {
@@ -2464,7 +2473,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
for i, c := range cands {
in := membersImageInput([]provider.Member{{Provider: c.p, Model: c.model}})
switch {
- case !blindTo(c.p.ID, c.model, in):
+ case !s.blindTo(c.p.ID, c.model, in):
sees, seesAt = append(sees, c), append(seesAt, pl.order[i])
case in == nil:
unknown, unknownAt = append(unknown, c), append(unknownAt, pl.order[i])
@@ -2731,7 +2740,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
}
if isGroup {
in := membersImageInput([]provider.Member{{Provider: c.p, Model: c.model}})
- if seenGroup != nil && blindTo(c.p.ID, c.model, in) {
+ if seenGroup != nil && s.blindTo(c.p.ID, c.model, in) {
groupUnknown = in == nil
b, err := seenGroup()
if err != nil {
@@ -3234,6 +3243,33 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
if plainFor == c.who()+"|"+c.model && call.Status > 0 && call.Status < 400 {
effortUpdatesRefused(c.who(), c.model)
}
+ if !hw.passing && !s.sightRefused(c.p.ID, c.model) && imageRefused(hw.code(), hw.errBody()) && hasImage(from, attemptBody) {
+ // the model turned the image away, its list or models.dev
+ // having said it sees: it is text-only from now on (blindTo),
+ // and with a model to describe images it is asked again with
+ // them described, before any other candidate — the user picked
+ // this one, and the Image recognition model is what they set
+ // for it. A pick of DeepSeek-V4.1-Flash at a relay serving it
+ // text-only went on to the same model at another provider
+ // (xyenon). With nothing to describe them it goes on as
+ // before, to a candidate that may see.
+ s.refuseSight(c.p.ID, c.model)
+ if isGroup && seenGroup != nil {
+ try.Fail = failSight
+ s.trace.update(tr, func(t *Route) { t.Tries[len(t.Tries)-1] = try })
+ i--
+ continue
+ }
+ if see, ok := seeing(); ok && !isGroup {
+ if seen, err := s.seenBody(withDescribeFor(r.Context(), call.Agent, call.Model, sessionOf(r.Header), false), from, body, see); err == nil {
+ body = seen
+ try.Fail = failSight
+ s.trace.update(tr, func(t *Route) { t.Tries[len(t.Tries)-1] = try })
+ i--
+ continue
+ }
+ }
+ }
if !floored && !hw.passing && hw.code() == 400 {
// asked for fewer tokens than this provider answers with (#64)
if b, ok := withTokenFloor(body, tokenFloor(hw.errBody())); ok {
diff --git a/internal/gateway/routing.go b/internal/gateway/routing.go
index 388292657..dd37d7c29 100644
--- a/internal/gateway/routing.go
+++ b/internal/gateway/routing.go
@@ -300,6 +300,9 @@ const (
// failFloor: the request asked for a shorter reply than the provider
// gives, and is sent again asking for the least it takes
failFloor = "floor"
+ // failSight: the model turned an image away (imageRefused), and is
+ // asked again with the request's images described (refuseSight)
+ failSight = "sight"
// failUpdate: the upstream turned away the configuration_update items
// a changed effort went as (#617), and the request is sent again
// without them
diff --git a/internal/gateway/vision.go b/internal/gateway/vision.go
index dc2dfd857..48200b08f 100644
--- a/internal/gateway/vision.go
+++ b/internal/gateway/vision.go
@@ -9,6 +9,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
+ "regexp"
"slices"
"strings"
"sync"
@@ -92,7 +93,16 @@ func describedFor(ctx context.Context) *CallFor {
// list says it takes none (in), or says nothing and magpie counts it
// text-only — as agents were told before Vision had them told every model
// takes images (provider.Described): models.dev doesn't say it sees either.
-func blindTo(pid, model string, in *bool) bool {
+//
+// One that turned an image away since magpie started (sightRefused) is
+// blind whatever its list or models.dev says: models.dev's DeepSeek-V4.1-
+// Flash takes images, and a relay serving it with text only refused the
+// image with a 400 that sent the request on to the same model at another
+// provider, past the Image recognition model the user picked (xyenon).
+func (s *Server) blindTo(pid, model string, in *bool) bool {
+ if s.sightRefused(pid, model) {
+ return true
+ }
if in != nil {
return !*in
}
@@ -493,3 +503,36 @@ func smallModelWhere(p provider.Provider, kind string, keep func(catalog.Model)
}
return c.id, c.cost, ready
}
+
+// imageRefusedWords are how a vendor turns away an image its model can't
+// take: DeepSeek's 400 "Failed to deserialize the JSON body into the
+// target type: messages[0]: unknown variant `image_url`, expected `text`"
+// (and Responses' input_image, Messages' image, so spelt), OpenAI's
+// "image_url is only supported by certain models", OpenRouter's "No
+// endpoints found that support image input", vLLM's "is not a multimodal
+// model", and the plain "does not support image input" / "image input is
+// not supported" / "不支持图片" of the rest.
+var imageRefusedWords = regexp.MustCompile("(?i)unknown variant [`'\"\\\\]*(?:image_url|input_image|image)\\b|image_url is only supported by certain models|no endpoints found that support image|not a multi-?modal model|(?:does not|doesn't|do not|don't|cannot|can't|unable to) (?:support|accept|process|handle|take|read) (?:the )?(?:image|images|vision|multi-?modal)|\\b(?:image|images|vision|multi-?modal|image_url|input_image)(?: (?:input|inputs|content|parts?))? (?:is |are )?(?:not supported|unsupported|not enabled|not allowed)|不支持(?:图片|图像|多模态|视觉|识图)")
+
+// imageRefused says a failure was the model turning the request's image
+// away: what a describer fixes by describing it, not what another
+// provider of the model should be sent instead (refuseSight).
+func imageRefused(status int, body []byte) bool {
+ switch status {
+ case http.StatusBadRequest, http.StatusUnsupportedMediaType, http.StatusUnprocessableEntity:
+ default:
+ return false
+ }
+ return imageRefusedWords.Match(wordsOf(body))
+}
+
+// refuseSight records that pid's model turned an image away (Server.blind):
+// each later image to it is described (blindTo) or, with nothing to
+// describe it, sent only to a model that sees.
+func (s *Server) refuseSight(pid, model string) { s.blind.Store(pid+"/"+model, true) }
+
+// sightRefused is whether pid's model turned an image away (refuseSight).
+func (s *Server) sightRefused(pid, model string) bool {
+ _, ok := s.blind.Load(pid + "/" + model)
+ return ok
+}
diff --git a/internal/gateway/vision_refused_test.go b/internal/gateway/vision_refused_test.go
new file mode 100644
index 000000000..8e231c210
--- /dev/null
+++ b/internal/gateway/vision_refused_test.go
@@ -0,0 +1,197 @@
+package gateway
+
+import (
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/yetone/magpie/internal/catalog"
+ "github.com/yetone/magpie/internal/provider"
+ "github.com/yetone/magpie/internal/settings"
+)
+
+// deepseekImageRefusal is DeepSeek's own 400 for an image sent to a model
+// that takes text only, as a relay serving DeepSeek-V4.1-Flash passes it on.
+const deepseekImageRefusal = `{"error":{"message":"Failed to deserialize the JSON body into the target type: messages[0]: unknown variant ` + "`image_url`" + `, expected ` + "`text`" + ` at line 1 column 162","type":"invalid_request_error","param":null,"code":"invalid_request_error"}}`
+
+// sightUp is an upstream serving one model: blind turns away every
+// request carrying an image with DeepSeek's 400; otherwise it answers ok.
+// What it is sent is kept.
+type sightUp struct {
+ mu sync.Mutex
+ blind bool
+ bodies []string
+}
+
+func (u *sightUp) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ body, _ := io.ReadAll(r.Body)
+ u.mu.Lock()
+ u.bodies = append(u.bodies, string(body))
+ u.mu.Unlock()
+ w.Header().Set("Content-Type", "application/json")
+ if u.blind && strings.Contains(string(body), "image_url") {
+ w.WriteHeader(400)
+ io.WriteString(w, deepseekImageRefusal)
+ return
+ }
+ if strings.Contains(string(body), `"stream":true`) {
+ w.Header().Set("Content-Type", "text/event-stream")
+ io.WriteString(w, `data: {"id":"x","choices":[{"index":0,"delta":{"role":"assistant","content":"ok"}}]}`+"\n\n")
+ io.WriteString(w, `data: {"id":"x","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1}}`+"\n\n")
+ io.WriteString(w, "data: [DONE]\n\n")
+ return
+ }
+ io.WriteString(w, `{"id":"x","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}`)
+}
+
+func (u *sightUp) sent() []string {
+ u.mu.Lock()
+ defer u.mu.Unlock()
+ return append([]string(nil), u.bodies...)
+}
+
+// sameModelTwice is xyenon's setup: DeepSeek-V4.1-Flash at two providers,
+// pa's a relay that takes text only but whose list says nothing of images
+// (models.dev says the model sees, so magpie counted it as seeing), pb's
+// one that sees; pa falls back to pb. The Image recognition model is
+// v/eye, unless vision is "" (magpie picks) or "off".
+func sameModelTwice(t *testing.T, vision string) (s *Server, pa, pb *sightUp, eye *eyeUp) {
+ t.Helper()
+ fresh(t)
+ pa, pb, eye = &sightUp{blind: true}, &sightUp{}, &eyeUp{}
+ sa, sb, se := httptest.NewServer(pa), httptest.NewServer(pb), httptest.NewServer(eye)
+ t.Cleanup(sa.Close)
+ t.Cleanup(sb.Close)
+ t.Cleanup(se.Close)
+ const m = "DeepSeek-V4.1-Flash"
+ save := func(p provider.Provider, ms []catalog.Model) {
+ t.Helper()
+ if err := provider.Save(p); err != nil {
+ t.Fatal(err)
+ }
+ if err := catalog.SaveLive(p.ID, p.Chat, ms); err != nil {
+ t.Fatal(err)
+ }
+ }
+ save(provider.Provider{ID: "pa", Name: "PA", Chat: sa.URL + "/v1", Key: "k", Models: []string{m}, Fallback: []string{"pb/" + m}},
+ []catalog.Model{{ID: m, Images: true}}) // models.dev's word, nothing of the relay's own
+ save(provider.Provider{ID: "pb", Name: "PB", Chat: sb.URL + "/v1", Key: "k", Models: []string{m}},
+ []catalog.Model{{ID: m, Images: true, ImageInput: imageInputBool(true)}})
+ save(provider.Provider{ID: "v", Name: "V", Chat: se.URL + "/v1", Key: "k", Models: []string{"eye"}},
+ []catalog.Model{{ID: "eye", Images: true, ImageInput: imageInputBool(true)}})
+ st := settings.Load()
+ st.Vision = vision
+ if err := settings.Save(st); err != nil {
+ t.Fatal(err)
+ }
+ return New(), pa, pb, eye
+}
+
+// imageTurns are a request with an image on each of the three APIs.
+func imageTurns(model string) []struct{ path, body string } {
+ return []struct{ path, body string }{
+ {"/v1/chat/completions", `{"model":"` + model + `","messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]}]}`},
+ {"/v1/responses", `{"model":"` + model + `","input":[{"role":"user","content":[{"type":"input_text","text":"read"},{"type":"input_image","image_url":"data:image/png;base64,aGVsbG8="}]}]}`},
+ {"/v1/messages", `{"model":"` + model + `","max_tokens":16,"messages":[{"role":"user","content":[{"type":"text","text":"read"},{"type":"image","source":{"type":"base64","media_type":"image/png","data":"aGVsbG8="}}]}]}`},
+ }
+}
+
+// The model the user picked, at the provider they picked, is given the
+// image described by the Image recognition model they set, when it turns
+// the image away: not the same model at another provider that sees, which
+// its fallback or its group reaches (xyenon on Discord). After the first
+// refusal it is described up front.
+func TestPickedModelThatRefusesAnImageIsGivenItDescribed(t *testing.T) {
+ for _, asked := range []string{"pa/DeepSeek-V4.1-Flash", "group/g", "group/auto-deepseek-v4-1-flash"} {
+ for _, tc := range imageTurns(asked) {
+ t.Run(asked+tc.path, func(t *testing.T) {
+ s, pa, pb, eye := sameModelTwice(t, "v/eye")
+ if asked == "group/g" {
+ if err := provider.SaveGroup(provider.Group{ID: "g", Name: "G", Members: []string{"pa/DeepSeek-V4.1-Flash", "pb/DeepSeek-V4.1-Flash"}, Routing: provider.Ordered}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for turn := range 2 {
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", tc.path, strings.NewReader(tc.body)))
+ if rec.Code != 200 {
+ t.Fatalf("turn %d: %d %s", turn, rec.Code, rec.Body.String())
+ }
+ }
+ if got := pb.sent(); len(got) != 0 {
+ t.Fatalf("the same model at the other provider was asked %d times: %s", len(got), got[0])
+ }
+ got := pa.sent()
+ // the first turn: the image, refused, then the description;
+ // the second turn: the description at once
+ if len(got) != 3 {
+ t.Fatalf("the picked provider was asked %d times, want 3: %q", len(got), got)
+ }
+ for _, b := range got[1:] {
+ if !strings.Contains(b, "HELLO") || !strings.Contains(b, "v/eye describes it") || strings.Contains(b, "aGVsbG8=") {
+ t.Fatalf("the picked provider was sent %s", b)
+ }
+ }
+ if n := len(eye.sent("eye")); n != 1 {
+ t.Fatalf("the image was described %d times, want 1", n)
+ }
+ })
+ }
+ }
+}
+
+// With nothing to describe images (Image recognition off), the same model
+// at a provider that sees is the only way to answer: the fallback is
+// asked, as before.
+func TestRefusedImageWithNoDescriberGoesToOneThatSees(t *testing.T) {
+ s, pa, pb, _ := sameModelTwice(t, "off")
+ tc := imageTurns("pa/DeepSeek-V4.1-Flash")[0]
+ for turn := range 2 {
+ rec := httptest.NewRecorder()
+ s.Handler().ServeHTTP(rec, httptest.NewRequest("POST", tc.path, strings.NewReader(tc.body)))
+ if rec.Code != 200 {
+ t.Fatalf("turn %d: %d %s", turn, rec.Code, rec.Body.String())
+ }
+ }
+ if n := len(pb.sent()); n != 2 {
+ t.Fatalf("the provider that sees was asked %d times, want 2", n)
+ }
+ if len(pa.sent()) == 0 {
+ t.Fatal("the picked provider was never asked")
+ }
+}
+
+func TestImageRefused(t *testing.T) {
+ yes := []string{
+ deepseekImageRefusal,
+ `{"error":{"message":"Failed to deserialize the JSON body into the target type: input[0].content[1]: unknown variant ` + "`input_image`" + `"}}`,
+ `{"error":{"message":"Invalid content type. image_url is only supported by certain models.","type":"invalid_request_error"}}`,
+ `{"error":{"message":"No endpoints found that support image input","code":404}}`,
+ `{"object":"error","message":"Qwen3-32B is not a multimodal model","type":"BadRequestError"}`,
+ `{"error":{"message":"This model does not support image input"}}`,
+ `{"error":{"message":"Image input is not supported for this model"}}`,
+ `{"error":{"message":"当前模型不支持图片输入"}}`,
+ }
+ for _, b := range yes {
+ if !imageRefused(400, []byte(b)) {
+ t.Errorf("not read as an image refused: %s", b)
+ }
+ }
+ no := []string{
+ `{"error":{"message":"Failed to deserialize the JSON body into the target type: tools[0]: unknown variant ` + "`custom`" + `"}}`,
+ `{"error":{"message":"You exceeded your current quota"}}`,
+ `{"error":{"message":"prompt is too long"}}`,
+ `{"error":{"message":"Insufficient balance"}}`,
+ }
+ for _, b := range no {
+ if imageRefused(400, []byte(b)) {
+ t.Errorf("read as an image refused: %s", b)
+ }
+ }
+ if imageRefused(429, []byte(deepseekImageRefusal)) || imageRefused(500, []byte(deepseekImageRefusal)) {
+ t.Error("a 429 or 500 read as an image refused")
+ }
+}
diff --git a/internal/gui/assets/plugins/routing/client.js b/internal/gui/assets/plugins/routing/client.js
index 2934c9345..224aaa4b8 100644
--- a/internal/gui/assets/plugins/routing/client.js
+++ b/internal/gui/assets/plugins/routing/client.js
@@ -508,7 +508,7 @@ function routingPage(part) {
const quota = (w, used, left, vars) => w.balance != null ? t(left, { n: balText(w), ...vars }) + lapses(w)
: (w.limit > 0 ? quotaCount(w) + " · " : "") + t(quotaLeft ? left : used, { n: pct(share(w)), ...vars });
const fill = (w) => Math.max(0, Math.min(100, share(w))) + "%";
- const FAIL = { rate: "rate limited", credit: "out of credit", quota: "quota used up", other: "failed", auth: "sign-in required", canceled: "canceled", foreign: "another account's reasoning", floor: "reply too short", verify: "needs verification", refused: "refused (safety filter)", shape: "request not understood", proxy: "proxy not reachable", effort: "reasoning effort not in its plan", overflow: "too long for its model", room: "reply asked past the window left", slow: "slow to start", prompt: "agent's prompt turned away", restricted: "account restricted by the vendor" };
+ const FAIL = { rate: "rate limited", credit: "out of credit", quota: "quota used up", other: "failed", auth: "sign-in required", canceled: "canceled", foreign: "another account's reasoning", floor: "reply too short", verify: "needs verification", refused: "refused (safety filter)", shape: "request not understood", sight: "images not taken", proxy: "proxy not reachable", effort: "reasoning effort not in its plan", overflow: "too long for its model", room: "reply asked past the window left", slow: "slow to start", prompt: "agent's prompt turned away", restricted: "account restricted by the vendor" };
const failWord = (why) => t(FAIL[why] || "failed");
// a try that answered: one whose reply broke off after it began (its 200
// sent, then the vendor's error, #733) didn't, and has a fail
@@ -1308,6 +1308,8 @@ function routingPage(part) {
return r.tries[i + 1]
? t("{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, so it goes on to the next before any of the reply reaches {agent}. Nothing is wrong with {who}, so it doesn't rest.", { who: name, status: tr.status, agent })
: t("{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, and nobody else is left to try, so {agent} gets the error. Nothing is wrong with {who}, so it doesn't rest.", { who: name, status: tr.status, agent });
+ if (tr.fail === "sight")
+ return t("{who} answered {status}: it takes no images, so it is asked again with the images described by the image recognition model, before any of the reply reaches {agent}. Later requests to it are described first. Nothing is wrong with {who}, so it doesn't rest.", { who: name, status: tr.status, agent });
if (tr.fail === "shape")
return t("{who} answered {status}: its API couldn't read something in the request that another's may, so it goes on to the next before any of the reply reaches {agent}. Nothing is wrong with {who}, so it doesn't rest.", { who: name, status: tr.status, agent });
if (tr.fail === "effort")
diff --git a/internal/gui/assets/plugins/routing/i18n.js b/internal/gui/assets/plugins/routing/i18n.js
index 3964cef6f..3a72ab5f9 100644
--- a/internal/gui/assets/plugins/routing/i18n.js
+++ b/internal/gui/assets/plugins/routing/i18n.js
@@ -359,6 +359,8 @@ magpie.strings({
"{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, so it goes on to the next before any of the reply reaches {agent}. Nothing is wrong with {who}, so it doesn't rest.": "{who} 返回 {status}:服务商不接受 {agent} 的系统提示词,换哪个账号都一样,已在回复到达 {agent} 前转给下一个。{who} 本身无问题,不休息。",
"{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, and nobody else is left to try, so {agent} gets the error. Nothing is wrong with {who}, so it doesn't rest.": "{who} 返回 {status}:服务商不接受 {agent} 的系统提示词,换哪个账号都一样,且没有其他可尝试的,{agent} 收到该错误。{who} 本身无问题,不休息。",
"request not understood": "请求格式不被接受",
+ "images not taken": "不接受图片",
+ "{who} answered {status}: it takes no images, so it is asked again with the images described by the image recognition model, before any of the reply reaches {agent}. Later requests to it are described first. Nothing is wrong with {who}, so it doesn't rest.": "{who} 返回 {status}:它不接受图片,已改用看图模型描述图片后重新询问,回复尚未到达 {agent}。之后发给它的请求会先描述图片。{who} 本身无问题,不休息。",
"reasoning effort not in its plan": "套餐不支持该推理强度",
"{who} answered {status}: its plan doesn't take the reasoning effort asked for, so another account goes on with it before any of the reply reaches {agent}. {who} serves other efforts, so it doesn't rest.": "{who} 返回 {status}:其套餐不支持所请求的推理强度,已在回复到达 {agent} 前转给另一个账号。{who} 仍支持其他强度,不休息。",
"{who} answered {status}: its plan doesn't take the reasoning effort asked for, and no account left that does could answer, so {agent} gets an error saying so.": "{who} 返回 {status}:其套餐不支持所请求的推理强度,也没有其他支持且能应答的账号,{agent} 收到说明原因的错误。",
@@ -946,6 +948,8 @@ magpie.strings({
"{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, so it goes on to the next before any of the reply reaches {agent}. Nothing is wrong with {who}, so it doesn't rest.": "{who} 返回 {status}:服務商不接受 {agent} 的系統提示詞,換哪個帳號都一樣,已在回覆到達 {agent} 前轉給下一個。{who} 本身無問題,不休息。",
"{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, and nobody else is left to try, so {agent} gets the error. Nothing is wrong with {who}, so it doesn't rest.": "{who} 返回 {status}:服務商不接受 {agent} 的系統提示詞,換哪個帳號都一樣,且沒有其他可嘗試的,{agent} 收到該錯誤。{who} 本身無問題,不休息。",
"request not understood": "請求格式不被接受",
+ "images not taken": "不接受圖片",
+ "{who} answered {status}: it takes no images, so it is asked again with the images described by the image recognition model, before any of the reply reaches {agent}. Later requests to it are described first. Nothing is wrong with {who}, so it doesn't rest.": "{who} 返回 {status}:它不接受圖片,已改用看圖模型描述圖片後重新詢問,回覆尚未到達 {agent}。之後發給它的請求會先描述圖片。{who} 本身無問題,不休息。",
"reasoning effort not in its plan": "套餐不支援該推理強度",
"{who} answered {status}: its plan doesn't take the reasoning effort asked for, so another account goes on with it before any of the reply reaches {agent}. {who} serves other efforts, so it doesn't rest.": "{who} 返回 {status}:其套餐不支援所請求的推理強度,已在回覆到達 {agent} 前轉給另一個帳號。{who} 仍支援其他強度,不休息。",
"{who} answered {status}: its plan doesn't take the reasoning effort asked for, and no account left that does could answer, so {agent} gets an error saying so.": "{who} 返回 {status}:其套餐不支援所請求的推理強度,也沒有其他支援且能應答的帳號,{agent} 收到說明原因的錯誤。",
@@ -1531,6 +1535,8 @@ magpie.strings({
"{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, so it goes on to the next before any of the reply reaches {agent}. Nothing is wrong with {who}, so it doesn't rest.": "{who} が {status} を返しました:ベンダーはどのアカウントでも {agent} のシステムプロンプトを拒否するため、応答が {agent} に届く前に次へ進みます。{who} に問題はないので休止しません。",
"{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, and nobody else is left to try, so {agent} gets the error. Nothing is wrong with {who}, so it doesn't rest.": "{who} が {status} を返しました:ベンダーはどのアカウントでも {agent} のシステムプロンプトを拒否し、他に試せる相手もいないため、{agent} にエラーを返します。{who} に問題はないので休止しません。",
"request not understood": "リクエストを理解できない",
+ "images not taken": "画像を受け付けない",
+ "{who} answered {status}: it takes no images, so it is asked again with the images described by the image recognition model, before any of the reply reaches {agent}. Later requests to it are described first. Nothing is wrong with {who}, so it doesn't rest.": "{who} が {status} を返しました:画像を受け付けないため、画像認識モデルが画像を説明したうえで、応答が {agent} に届く前にもう一度尋ねます。以後のリクエストは先に画像を説明します。{who} に問題はないので休止しません。",
"reasoning effort not in its plan": "プランにない推論レベル",
"{who} answered {status}: its plan doesn't take the reasoning effort asked for, so another account goes on with it before any of the reply reaches {agent}. {who} serves other efforts, so it doesn't rest.": "{who} が {status} を返しました:そのプランは指定された推論レベルに対応していないため、応答が {agent} に届く前に別のアカウントが引き継ぎます。{who} は他の推論レベルには対応しているので、休止しません。",
"{who} answered {status}: its plan doesn't take the reasoning effort asked for, and no account left that does could answer, so {agent} gets an error saying so.": "{who} が {status} を返しました:そのプランは指定された推論レベルに対応しておらず、対応していて応答できるアカウントも残っていないため、{agent} にはその旨のエラーが返ります。",
@@ -2118,6 +2124,8 @@ magpie.strings({
"{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, so it goes on to the next before any of the reply reaches {agent}. Nothing is wrong with {who}, so it doesn't rest.": "{who} antwortete mit {status}: Der Anbieter lehnt den Systemprompt von {agent} bei jedem Konto ab, daher geht sie an den nächsten, bevor etwas von der Antwort {agent} erreicht. Mit {who} ist alles in Ordnung, daher pausiert es nicht.",
"{who} answered {status}: the vendor turns away {agent}'s system prompt whichever account it goes to, and nobody else is left to try, so {agent} gets the error. Nothing is wrong with {who}, so it doesn't rest.": "{who} antwortete mit {status}: Der Anbieter lehnt den Systemprompt von {agent} bei jedem Konto ab, und niemand sonst ist übrig, daher bekommt {agent} den Fehler. Mit {who} ist alles in Ordnung, daher pausiert es nicht.",
"request not understood": "Anfrage nicht verstanden",
+ "images not taken": "nimmt keine Bilder",
+ "{who} answered {status}: it takes no images, so it is asked again with the images described by the image recognition model, before any of the reply reaches {agent}. Later requests to it are described first. Nothing is wrong with {who}, so it doesn't rest.": "{who} antwortete mit {status}: Es nimmt keine Bilder, daher wird es erneut gefragt, mit den Bildern vom Bilderkennungsmodell beschrieben, bevor etwas von der Antwort {agent} erreicht. Spätere Anfragen werden zuerst beschrieben. Mit {who} ist alles in Ordnung, daher pausiert es nicht.",
"reasoning effort not in its plan": "Reasoning-Effort nicht im Plan",
"{who} answered {status}: its plan doesn't take the reasoning effort asked for, so another account goes on with it before any of the reply reaches {agent}. {who} serves other efforts, so it doesn't rest.": "{who} antwortete mit {status}: Sein Plan unterstützt den verlangten Reasoning-Effort nicht, daher übernimmt ein anderes Konto, bevor etwas von der Antwort {agent} erreicht. {who} bedient andere Effort-Stufen und pausiert daher nicht.",
"{who} answered {status}: its plan doesn't take the reasoning effort asked for, and no account left that does could answer, so {agent} gets an error saying so.": "{who} antwortete mit {status}: Sein Plan unterstützt den verlangten Reasoning-Effort nicht, und kein verbleibendes Konto, das ihn unterstützt, konnte antworten, daher erhält {agent} einen entsprechenden Fehler.",
diff --git a/internal/gui/tests/routing-sight.test.cjs b/internal/gui/tests/routing-sight.test.cjs
new file mode 100644
index 000000000..2979b7bc7
--- /dev/null
+++ b/internal/gui/tests/routing-sight.test.cjs
@@ -0,0 +1,112 @@
+// Run with Node's test runner and Playwright on the module path; see README.md.
+// A model that turned an image away (xyenon: a relay's DeepSeek-V4.1-Flash
+// answering 400 "unknown variant `image_url`") is asked again with the image
+// described, not handed to another provider: the Routing page's story says
+// so, in the page's language, and that it went on to no other provider.
+const assert = require("node:assert/strict");
+const fs = require("node:fs/promises");
+const path = require("node:path");
+const { test } = require("node:test");
+const { chromium, webkit } = require("playwright");
+
+const assets = path.resolve(__dirname, "../assets");
+const now = new Date();
+const day = [now.getFullYear(), now.getMonth() + 1, now.getDate()].map((n) => String(n).padStart(2, "0")).join("-");
+const at = (i) => new Date(now.getTime() - (i + 1) * 60e3).toISOString();
+const vendor = "provider-1: Failed to deserialize the JSON body into the target type: messages[1]: unknown variant `image_url`, expected `text` at line 1 column 162";
+const p1 = { id: "provider-1", provider: "provider-1", name: "provider-1", kind: "provider", model: "DeepSeek-V4.1-Flash" };
+const p2 = { id: "provider-2", provider: "provider-2", name: "provider-2", kind: "provider", model: "DeepSeek-V4.1-Flash" };
+const routes = [{
+ id: 100, seq: 100, time: at(0), agent: "codex", model: "provider-1/DeepSeek-V4.1-Flash", provider: "provider-1", order: [p1, p2],
+ tries: [
+ { id: p1.id, model: p1.model, start: at(0), done: true, status: 400, ms: 90, fail: "sight", error: vendor },
+ { id: p1.id, model: p1.model, start: at(0), done: true, status: 200, ms: 800 },
+ ],
+ done: true, status: 200, ms: 900,
+}, {
+ id: 99, seq: 99, time: at(1), agent: "codex", model: "provider-1/DeepSeek-V4.1-Flash", provider: "provider-1", order: [p1, p2],
+ tries: [{ id: p1.id, model: p1.model, start: at(1), done: true, status: 200, ms: 500 }], done: true, status: 200, ms: 500,
+}];
+
+function serve(lang) {
+ const state = { agents: [{ id: "codex", name: "Codex", path: "/test/config.toml", fields: [] }], profiles: [], settings: { lang, theme: "light" } };
+ return async (route) => {
+ const url = new URL(route.request().url());
+ const json = (data) => route.fulfill({ json: data });
+ if (url.pathname === "/boot.js") return route.fulfill({ contentType: "text/javascript", body: `window.bootPrefs = {lang:"${lang}",theme:"light",web:true};` });
+ if (url.pathname === "/wails/runtime.js") return route.fulfill({ contentType: "text/javascript", body: "export const Window = {};" });
+ if (url.pathname === "/api/state") return json(state);
+ if (url.pathname === "/api/plugins") return json({ plugins: [] });
+ if (url.pathname === "/api/gateway/trace") {
+ if (url.searchParams.get("wait")) await new Promise((r) => setTimeout(r, 20e3));
+ return json({ mine: true, now: now.toISOString(), seq: 1, totals: { requests: 0, rerouted: 0, errors: 0 }, routes: [] });
+ }
+ if (url.pathname === "/api/gateway/history") {
+ const d = url.searchParams.get("day");
+ return json({ cut: false, days: [{ day, requests: routes.length }], routes: d ? routes : [] });
+ }
+ if (url.pathname === "/api/groups") return json({ groups: [] });
+ if (url.pathname === "/api/providers") return json({ providers: [], gateway: { running: true, window: true } });
+ if (url.pathname.startsWith("/api/")) return json({});
+ const file = path.join(assets, url.pathname === "/" ? "index.html" : url.pathname);
+ const contentType = { ".html": "text/html", ".js": "text/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".png": "image/png" }[path.extname(file)];
+ await route.fulfill({ body: await fs.readFile(file), contentType });
+ };
+}
+
+const want = {
+ en: { story: "it takes no images, so it is asked again with the images described by the image recognition model", rest: "so it doesn't rest" },
+ zh: { story: "它不接受图片,已改用看图模型描述图片后重新询问", rest: "不休息" },
+ "zh-TW": { story: "它不接受圖片,已改用看圖模型描述圖片後重新詢問", rest: "不休息" },
+ ja: { story: "画像を受け付けないため、画像認識モデルが画像を説明したうえで", rest: "休止しません" },
+ de: { story: "Es nimmt keine Bilder, daher wird es erneut gefragt", rest: "pausiert es nicht" },
+};
+
+for (const engine of (process.env.BROWSER ? [process.env.BROWSER] : ["chromium", "webkit"])) {
+ for (const lang of Object.keys(want)) {
+ test(`${engine} ${lang}: a model that took no image is asked again with it described`, async (t) => {
+ const browser = await (engine === "webkit" ? webkit.launch() : chromium.launch({ channel: "chromium" }));
+ const context = await browser.newContext({ viewport: { width: 1100, height: 760 }, reducedMotion: "reduce" });
+ const page = await context.newPage();
+ page.setDefaultTimeout(5000);
+ const errors = [];
+ page.on("pageerror", (e) => errors.push(e.message));
+ await page.route("**/*", serve(lang));
+ t.after(async () => {
+ if (process.env.ARTIFACT_DIR) {
+ await fs.mkdir(process.env.ARTIFACT_DIR, { recursive: true });
+ await page.screenshot({ path: path.join(process.env.ARTIFACT_DIR, `${engine}-${lang}-sight.png`), fullPage: true });
+ }
+ await browser.close();
+ });
+ await page.goto("http://magpie.test/?view=routing");
+ await page.locator(".rt-days .rt-day").nth(1).click();
+ await page.locator(".rt-req").nth(routes.length - 1).waitFor();
+
+ const row = page.locator(".rt-req").nth(0);
+ // measure once the page holds still: the day's rows arriving grow the
+ // page above the list, and the day button's click is kept in place by
+ // scrolling the view, which can land after the last row is in
+ const was = await row.evaluate(async (e) => {
+ let last = NaN;
+ for (let i = 0; i < 60; i++) {
+ await new Promise((r) => requestAnimationFrame(r));
+ const top = e.getBoundingClientRect().top;
+ if (top === last) return top;
+ last = top;
+ }
+ return last;
+ });
+ await row.click();
+ await page.waitForTimeout(400);
+ assert(Math.abs((await row.evaluate((e) => e.getBoundingClientRect().top)) - was) <= 1, "picking the request moved the page");
+ const steps = await page.locator(".rt-steps li").allTextContents();
+ const story = steps.find((s) => s.includes(want[lang].story));
+ assert(story, JSON.stringify(steps));
+ assert(story.includes(want[lang].rest) && story.includes("400"), story);
+ // the same model answered after it: its row says so, not a failure
+ assert(!steps.some((s) => s.includes("went on to the next") || s.includes("转给下一个")), JSON.stringify(steps));
+ assert.deepEqual(errors, []);
+ });
+ }
+}