Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
gateway: a request naming no reply limit asks DeepSeek, and a translated one any vendor, for the model's own
yetoned157700a0a5c
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.goindex 85937d40e..f9da9b196 100644--- a/internal/gateway/gateway.go+++ b/internal/gateway/gateway.go@@ -2620,6 +2620,10 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro} else if n := userOutput(c.p, c.model); n > 0 {attemptBody = withMaxOutput(from, attemptBody, n)}+ // a request naming no reply limit, to an API that would cut it+ // short (DeepSeek's 8K), asks for the model's own, relayed or+ // translated (withOwnOutput)+ attemptBody = withOwnOutput(c.p, c.model, from, attemptBody)if roomFor == c.who()+"|"+c.model {if b, ok := withReplyCap(from, attemptBody, roomCap); ok {attemptBody = b@@ -5294,6 +5298,9 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, tovar beforeOff *Requestvar toolsRefusal []byteoffAsked := false+ // the reply limit was filled in, the client having named none+ // (fillsOutput)+ filledOutput := falsefor {if mayDropImageTool(p) && !s.fits(p.ID, imageToolRefused, to) {req, _ = withoutImageToolReq(req)@@ -5408,11 +5415,15 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, tor.Ultrafast, req = true, &r}// Anthropic's API needs a reply limit: one the client didn't name- // is the model's own, not a flat 16384 (#1792, askedOutput)- if to == provider.Anthropic && req.MaxTokens <= 0 {+ // is the model's own, not a flat 16384 (#1792, askedOutput). So on+ // a vendor's Chat or Responses, which may read none as less than+ // the model gives (fillsOutput), unless it turned the one filled+ // in away before+ if req.MaxTokens <= 0 && fillsOutput(p, to) && (to == provider.Anthropic || s.fits(p.ID, outputFillRefused(model), to)) {if n := askedOutput(p, model); n > 0 {r := *reqr.MaxTokens, req = n, &r+ filledOutput = to != provider.Anthropic}}wire := provider.UpstreamNameIn(wires, p.ID, model)@@ -5486,6 +5497,16 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, tob, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))res.Body.Close()res.Body = io.NopCloser(bytes.NewReader(b))+ if filledOutput && outputFieldRefused(res.StatusCode, b) {+ // the model's own limit, filled in, turned away (a list or+ // models.dev saying more than the vendor takes): asked again+ // naming none, as the client did, and so from then on+ log.Printf("gateway: %s turned away max tokens %d for %s (%d); asked naming none from now on", p.ID, req.MaxTokens, model, res.StatusCode)+ s.markUnfit(p.ID, outputFillRefused(model), to)+ r := *req+ r.MaxTokens, req, filledOutput = 0, &r, false+ continue+ }if req.LoadedInPlace && refusesField(res.StatusCode, b, additionalToolsItem) && bytes.Contains(body, []byte(`"`+additionalToolsItem+`"`)) {// an upstream that turns the item away is asked again with the// loaded tools among the others, and not sent it again: thediff --git a/internal/gateway/maxoutput.go b/internal/gateway/maxoutput.goindex 67421c341..3e8b783fd 100644--- a/internal/gateway/maxoutput.go+++ b/internal/gateway/maxoutput.go@@ -5,6 +5,7 @@ import ("encoding/json""regexp""strconv"+ "strings""github.com/tidwall/gjson"@@ -179,9 +180,105 @@ func withReplyCap(proto provider.Protocol, body []byte, limit int) ([]byte, boolreturn nil, false}key = "max_output_tokens"+ case provider.Gemini:+ gc := gjson.GetBytes(body, "generationConfig")+ if gc.Get("maxOutputTokens").Exists() {+ return nil, false+ }+ m := map[string]any{}+ if gc.IsObject() {+ dec := json.NewDecoder(strings.NewReader(gc.Raw))+ dec.UseNumber()+ if dec.Decode(&m) != nil {+ return nil, false+ }+ }+ m["maxOutputTokens"] = limit+ return withFields(body, map[string]any{"generationConfig": m}), truedefault:- // Anthropic's always names one; Gemini's is lowered above+ // Anthropic's always names onereturn nil, false}return withFields(body, map[string]any{key: limit}), true}++// namesOutput is whether body, in proto's words, names a reply limit.+func namesOutput(proto provider.Protocol, body []byte) bool {+ switch proto {+ case provider.Chat:+ return gjson.GetBytes(body, "max_tokens").Exists() || gjson.GetBytes(body, "max_completion_tokens").Exists()+ case provider.Responses:+ return gjson.GetBytes(body, "max_output_tokens").Exists()+ case provider.Anthropic:+ return gjson.GetBytes(body, "max_tokens").Exists()+ case provider.Gemini:+ return gjson.GetBytes(body, "generationConfig.maxOutputTokens").Exists()+ }+ return true+}++// shortByDefault is whether p's API cuts a reply that names no limit far+// short of what its model can give: DeepSeek's gives 8K without thinking+// and 64K with it (api-docs.deepseek.com, create-chat-completion:+// "When not set, the default is 8K in non-thinking mode, 64K in thinking+// mode"), on /v1/responses too, where its models go to 384K. Codex names+// no max_output_tokens, so its turns there stopped at 8192 tokens, "Incomplete+// response returned, reason: max_output_tokens" (xuanwo on X). Checked+// against api.deepseek.com on 2026-10-27: deepseek-flash, effort none, no+// limit: incomplete at 8192; max_output_tokens 393216: completed at 17602.+func shortByDefault(p provider.Provider) bool {+ return p.Host() == "api.deepseek.com"+}++// withOwnOutput is body, in proto's words, asking p's model for the reply+// its own limit allows (askedOutput) when it names none and p's API would+// otherwise cut it short (shortByDefault). A limit the client named goes+// as it is.+func withOwnOutput(p provider.Provider, model string, proto provider.Protocol, body []byte) []byte {+ if !shortByDefault(p) || namesOutput(proto, body) {+ return body+ }+ n := askedOutput(p, model)+ if n <= 0 {+ return body+ }+ if b, ok := withReplyCap(proto, body, n); ok {+ return b+ }+ return body+}++// fillsOutput is whether a request translated for p's API on to that+// names no reply limit is sent its model's own (askedOutput): the client's+// API reads none as the model's whole, which another vendor's needn't+// (DeepSeek's reads it as 8K, shortByDefault). Anthropic's always is, as+// it needs one. An account's own backend (the ChatGPT one turns+// max_output_tokens away) and another magpie, which fills its own, aren't.+func fillsOutput(p provider.Provider, to provider.Protocol) bool {+ if to == provider.Anthropic {+ return true+ }+ return (to == provider.Chat || to == provider.Responses) &&+ (p.Account == nil || p.IsPlugin()) && !p.IsRemoteMagpie()+}++// outputFieldRefused is a request error about the reply limit itself+// ("max_tokens must be at most 32768"), not the window it shares with the+// prompt, which replyLeft reads and the request is asked again for.+func outputFieldRefused(status int, body []byte) bool {+ if !badRequest(status) {+ return false+ }+ w := wordsOf(body)+ if replyLeft(w) > 0 {+ return false+ }+ m := strings.ToLower(string(w))+ return strings.Contains(m, "max_tokens") || strings.Contains(m, "max_output_tokens") ||+ strings.Contains(m, "max_completion_tokens") || strings.Contains(m, "maxoutputtokens")+}++// outputFillRefused names, for markUnfit, a model whose vendor turned+// away the reply limit magpie filled in: a request naming none is sent+// none there from then on, as before.+func outputFillRefused(model string) string { return "filled-output:" + model }diff --git a/internal/gateway/own_output_test.go b/internal/gateway/own_output_test.gonew file mode 100644index 000000000..20323a8ea--- /dev/null+++ b/internal/gateway/own_output_test.go@@ -0,0 +1,205 @@+package gateway++import (+ "context"+ "encoding/json"+ "io"+ "net"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strings"+ "sync"+ "testing"++ "github.com/yetone/magpie/internal/catalog"+ "github.com/yetone/magpie/internal/provider"+)++// ownOutputUpstream answers every API magpie may ask on, whole, and keeps+// each body it was sent. A Chat body naming max_tokens past refuseOver is+// turned away as a vendor whose limit is lower than its list says does.+type ownOutputUpstream struct {+ mu sync.Mutex+ got []map[string]any+ refuseOver int+}++func (u *ownOutputUpstream) ServeHTTP(w http.ResponseWriter, r *http.Request) {+ b, _ := io.ReadAll(r.Body)+ var q map[string]any+ json.Unmarshal(b, &q)+ u.mu.Lock()+ u.got = append(u.got, q)+ refuse := u.refuseOver+ u.mu.Unlock()+ stream := q["stream"] == true+ if stream {+ w.Header().Set("Content-Type", "text/event-stream")+ } else {+ w.Header().Set("Content-Type", "application/json")+ }+ switch {+ case strings.HasSuffix(r.URL.Path, "/responses") && stream:+ io.WriteString(w, "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"r\",\"status\":\"in_progress\",\"output\":[]}}\n\n"++ "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"ok\"}\n\n"++ "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"r\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"o\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"ok\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n")+ case strings.HasSuffix(r.URL.Path, "/responses"):+ io.WriteString(w, `{"id":"r","object":"response","status":"completed","model":"m","output":[{"type":"message","id":"o","role":"assistant","status":"completed","content":[{"type":"output_text","text":"ok","annotations":[]}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}`)+ default:+ if n, _ := q["max_tokens"].(float64); refuse > 0 && int(n) > refuse {+ w.WriteHeader(400)+ io.WriteString(w, `{"error":{"message":"Invalid max_tokens value, the valid range of max_tokens is [1, 32768]","type":"invalid_request_error"}}`)+ return+ }+ if stream {+ io.WriteString(w, "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\n"++ "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\n"++ "data: [DONE]\n\n")+ return+ }+ io.WriteString(w, `{"id":"c","object":"chat.completion","model":"m","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)+ }+}++func (u *ownOutputUpstream) take() []map[string]any {+ u.mu.Lock()+ defer u.mu.Unlock()+ got := u.got+ u.got = nil+ return got+}++// Codex names no max_output_tokens, and DeepSeek's API reads none as 8K+// without thinking, 64K with it, where its models give 384K: Codex's turns+// on DeepSeek stopped there, "Incomplete response returned, reason:+// max_output_tokens" (xuanwo on X). A request naming no reply limit is+// sent the model's own there, relayed or translated, on every API a client+// may speak; a limit the client named goes as it is, and the same request+// to another vendor's Responses or Chat, relayed, goes as it came.+func TestShortDefaultAsksTheModelsOutput(t *testing.T) {+ fresh(t)+ u := &ownOutputUpstream{}+ up := httptest.NewServer(u)+ t.Cleanup(up.Close)+ os.MkdirAll(filepath.Dir(catalog.CachePath()), 0o755)+ if err := os.WriteFile(catalog.CachePath(), []byte(`{"deepseek":{"id":"deepseek","models":{+ "deepseek-flash":{"id":"deepseek-flash","limit":{"context":1048576,"output":393216}}}}}`), 0o644); err != nil {+ t.Fatal(err)+ }+ catalog.Reset()+ t.Cleanup(catalog.Reset)+ for _, p := range []provider.Provider{+ {ID: "deepseek", Name: "DeepSeek", Key: "k", Catalog: "deepseek", Models: []string{"deepseek-flash"},+ Chat: "http://api.deepseek.com/v1", Responses: "http://api.deepseek.com/v1"},+ // the same model on a relay that speaks Responses and Chat: its+ // default isn't DeepSeek's+ {ID: "relay", Name: "Relay", Key: "k", Catalog: "deepseek", Models: []string{"deepseek-flash"},+ Chat: "http://relay.example.com/v1", Responses: "http://relay.example.com/v1"},+ } {+ if err := provider.Save(p); err != nil {+ t.Fatal(err)+ }+ }+ s := New()+ addr := strings.TrimPrefix(up.URL, "http://")+ s.client = &http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {+ return (&net.Dialer{}).DialContext(ctx, network, addr)+ }}}+ const chat = `{"model":"%s","messages":[{"role":"user","content":"hi"}]}`+ for _, c := range []struct {+ name, path, body, field string+ want int // 0: the field isn't sent+ }{+ {"Codex's turn, relayed", "/v1/responses", `{"model":"deepseek/deepseek-flash","input":"hi"}`, "max_output_tokens", 393216},+ {"Codex's turn, its own limit", "/v1/responses", `{"model":"deepseek/deepseek-flash","input":"hi","max_output_tokens":1000}`, "max_output_tokens", 1000},+ {"a Chat client, relayed", "/v1/chat/completions", strings.Replace(chat, "%s", "deepseek/deepseek-flash", 1), "max_tokens", 393216},+ {"a Chat client, its own limit", "/v1/chat/completions", `{"model":"deepseek/deepseek-flash","max_completion_tokens":2000,"messages":[{"role":"user","content":"hi"}]}`, "max_tokens", 2000},+ {"a Gemini client, translated", "/v1beta/models/deepseek/deepseek-flash:generateContent", `{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`, "", 393216},+ // another host's Responses and Chat, relayed: as they came+ {"a relay's Responses", "/v1/responses", `{"model":"relay/deepseek-flash","input":"hi"}`, "max_output_tokens", 0},+ {"a relay's Chat", "/v1/chat/completions", strings.Replace(chat, "%s", "relay/deepseek-flash", 1), "max_tokens", 0},+ } {+ if code, raw := postJSON(t, s, c.path, c.body); code != 200 {+ t.Fatalf("%s: %d %s", c.name, code, raw)+ }+ got := u.take()+ if len(got) != 1 {+ t.Fatalf("%s: %d requests upstream", c.name, len(got))+ }+ field := c.field+ if field == "" {+ // translated: whichever API magpie spoke+ for _, k := range []string{"max_tokens", "max_completion_tokens", "max_output_tokens"} {+ if _, ok := got[0][k]; ok {+ field = k+ }+ }+ }+ n, sent := got[0][field].(float64)+ if c.want == 0 && sent || c.want != 0 && int(n) != c.want {+ t.Errorf("%s: %s %v (sent %v), want %d: %v", c.name, field, n, sent, c.want, got[0])+ }+ }+}++// Translated onto another vendor's Chat or Responses, a request naming no+// reply limit is sent its model's own, as on Anthropic's (#1792): the+// client's API reads none as the model's whole, which the vendor's needn't.+// A vendor that turns the figure away (a list saying more than it takes) is+// asked again naming none, as the client did, and from then on.+func TestTranslatedAsksTheModelsOutput(t *testing.T) {+ fresh(t)+ u := &ownOutputUpstream{}+ up := httptest.NewServer(u)+ t.Cleanup(up.Close)+ os.MkdirAll(filepath.Dir(catalog.CachePath()), 0o755)+ if err := os.WriteFile(catalog.CachePath(), []byte(`{"kimi":{"id":"kimi","models":{+ "k-big":{"id":"k-big","limit":{"context":262144,"output":65536}}}}}`), 0o644); err != nil {+ t.Fatal(err)+ }+ catalog.Reset()+ t.Cleanup(catalog.Reset)+ // a vendor serving Chat alone: Codex's turn is translated+ if err := provider.Save(provider.Provider{ID: "v", Name: "V", Key: "k", Catalog: "kimi", Models: []string{"k-big"}, Chat: up.URL + "/v1"}); err != nil {+ t.Fatal(err)+ }+ s := New()+ codex := `{"model":"v/k-big","input":"hi"}`+ ask := func() []map[string]any {+ t.Helper()+ if code, raw := postJSON(t, s, "/v1/responses", codex); code != 200 {+ t.Fatalf("%d %s", code, raw)+ }+ return u.take()+ }+ got := ask()+ if len(got) != 1 || got[0]["max_tokens"] != float64(65536) {+ t.Fatalf("translated onto Chat: %v, want max_tokens 65536", got)+ }+ if code, _ := postJSON(t, s, "/v1/responses", `{"model":"v/k-big","input":"hi","max_output_tokens":700}`); code != 200 {+ t.Fatal(code)+ }+ if got := u.take(); len(got) != 1 || got[0]["max_tokens"] != float64(700) {+ t.Fatalf("the client's own limit: %v, want max_tokens 700", got)+ }++ u.mu.Lock()+ u.refuseOver = 32768+ u.mu.Unlock()+ got = ask()+ if len(got) != 2 || got[0]["max_tokens"] != float64(65536) {+ t.Fatalf("refused: %v, want 65536 then none", got)+ }+ if _, ok := got[1]["max_tokens"]; ok {+ t.Fatalf("asked again with a limit: %v", got[1])+ }+ got = ask()+ if len(got) != 1 {+ t.Fatalf("after a refusal: %d requests, want 1 naming none", len(got))+ }+ if _, ok := got[0]["max_tokens"]; ok {+ t.Fatalf("filled in again after the vendor turned it away: %v", got[0])+ }+}