Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a request naming no reply limit asks DeepSeek, and a translated one any vendor, for the model's own

yetoned157700a0a5c
diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go
index 85937d40e..f9da9b196 100644
--- a/internal/gateway/gateway.go
+++ b/internal/gateway/gateway.go
@@ -2620,6 +2620,10 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, from provider.Pro
} else if n := userOutput(c.p, c.model); n > 0 {
attemptBody = withMaxOutput(from, attemptBody, n)
}
+ // a request naming no reply limit, to an API that would cut it
+ // short (DeepSeek's 8K), asks for the model's own, relayed or
+ // translated (withOwnOutput)
+ attemptBody = withOwnOutput(c.p, c.model, from, attemptBody)
if roomFor == c.who()+"|"+c.model {
if b, ok := withReplyCap(from, attemptBody, roomCap); ok {
attemptBody = b
@@ -5294,6 +5298,9 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, to
var beforeOff *Request
var toolsRefusal []byte
offAsked := false
+ // the reply limit was filled in, the client having named none
+ // (fillsOutput)
+ filledOutput := false
for {
if mayDropImageTool(p) && !s.fits(p.ID, imageToolRefused, to) {
req, _ = withoutImageToolReq(req)
@@ -5408,11 +5415,15 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, to
r.Ultrafast, req = true, &r
}
// Anthropic's API needs a reply limit: one the client didn't name
- // is the model's own, not a flat 16384 (#1792, askedOutput)
- if to == provider.Anthropic && req.MaxTokens <= 0 {
+ // is the model's own, not a flat 16384 (#1792, askedOutput). So on
+ // a vendor's Chat or Responses, which may read none as less than
+ // the model gives (fillsOutput), unless it turned the one filled
+ // in away before
+ if req.MaxTokens <= 0 && fillsOutput(p, to) && (to == provider.Anthropic || s.fits(p.ID, outputFillRefused(model), to)) {
if n := askedOutput(p, model); n > 0 {
r := *req
r.MaxTokens, req = n, &r
+ filledOutput = to != provider.Anthropic
}
}
wire := provider.UpstreamNameIn(wires, p.ID, model)
@@ -5486,6 +5497,16 @@ func (s *Server) forwardTranslated(ctx context.Context, p provider.Provider, to
b, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20))
res.Body.Close()
res.Body = io.NopCloser(bytes.NewReader(b))
+ if filledOutput && outputFieldRefused(res.StatusCode, b) {
+ // the model's own limit, filled in, turned away (a list or
+ // models.dev saying more than the vendor takes): asked again
+ // naming none, as the client did, and so from then on
+ log.Printf("gateway: %s turned away max tokens %d for %s (%d); asked naming none from now on", p.ID, req.MaxTokens, model, res.StatusCode)
+ s.markUnfit(p.ID, outputFillRefused(model), to)
+ r := *req
+ r.MaxTokens, req, filledOutput = 0, &r, false
+ continue
+ }
if req.LoadedInPlace && refusesField(res.StatusCode, b, additionalToolsItem) && bytes.Contains(body, []byte(`"`+additionalToolsItem+`"`)) {
// an upstream that turns the item away is asked again with the
// loaded tools among the others, and not sent it again: the
diff --git a/internal/gateway/maxoutput.go b/internal/gateway/maxoutput.go
index 67421c341..3e8b783fd 100644
--- a/internal/gateway/maxoutput.go
+++ b/internal/gateway/maxoutput.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"regexp"
"strconv"
+ "strings"
"github.com/tidwall/gjson"
@@ -179,9 +180,105 @@ func withReplyCap(proto provider.Protocol, body []byte, limit int) ([]byte, bool
return nil, false
}
key = "max_output_tokens"
+ case provider.Gemini:
+ gc := gjson.GetBytes(body, "generationConfig")
+ if gc.Get("maxOutputTokens").Exists() {
+ return nil, false
+ }
+ m := map[string]any{}
+ if gc.IsObject() {
+ dec := json.NewDecoder(strings.NewReader(gc.Raw))
+ dec.UseNumber()
+ if dec.Decode(&m) != nil {
+ return nil, false
+ }
+ }
+ m["maxOutputTokens"] = limit
+ return withFields(body, map[string]any{"generationConfig": m}), true
default:
- // Anthropic's always names one; Gemini's is lowered above
+ // Anthropic's always names one
return nil, false
}
return withFields(body, map[string]any{key: limit}), true
}
+
+// namesOutput is whether body, in proto's words, names a reply limit.
+func namesOutput(proto provider.Protocol, body []byte) bool {
+ switch proto {
+ case provider.Chat:
+ return gjson.GetBytes(body, "max_tokens").Exists() || gjson.GetBytes(body, "max_completion_tokens").Exists()
+ case provider.Responses:
+ return gjson.GetBytes(body, "max_output_tokens").Exists()
+ case provider.Anthropic:
+ return gjson.GetBytes(body, "max_tokens").Exists()
+ case provider.Gemini:
+ return gjson.GetBytes(body, "generationConfig.maxOutputTokens").Exists()
+ }
+ return true
+}
+
+// shortByDefault is whether p's API cuts a reply that names no limit far
+// short of what its model can give: DeepSeek's gives 8K without thinking
+// and 64K with it (api-docs.deepseek.com, create-chat-completion:
+// "When not set, the default is 8K in non-thinking mode, 64K in thinking
+// mode"), on /v1/responses too, where its models go to 384K. Codex names
+// no max_output_tokens, so its turns there stopped at 8192 tokens, "Incomplete
+// response returned, reason: max_output_tokens" (xuanwo on X). Checked
+// against api.deepseek.com on 2026-10-27: deepseek-flash, effort none, no
+// limit: incomplete at 8192; max_output_tokens 393216: completed at 17602.
+func shortByDefault(p provider.Provider) bool {
+ return p.Host() == "api.deepseek.com"
+}
+
+// withOwnOutput is body, in proto's words, asking p's model for the reply
+// its own limit allows (askedOutput) when it names none and p's API would
+// otherwise cut it short (shortByDefault). A limit the client named goes
+// as it is.
+func withOwnOutput(p provider.Provider, model string, proto provider.Protocol, body []byte) []byte {
+ if !shortByDefault(p) || namesOutput(proto, body) {
+ return body
+ }
+ n := askedOutput(p, model)
+ if n <= 0 {
+ return body
+ }
+ if b, ok := withReplyCap(proto, body, n); ok {
+ return b
+ }
+ return body
+}
+
+// fillsOutput is whether a request translated for p's API on to that
+// names no reply limit is sent its model's own (askedOutput): the client's
+// API reads none as the model's whole, which another vendor's needn't
+// (DeepSeek's reads it as 8K, shortByDefault). Anthropic's always is, as
+// it needs one. An account's own backend (the ChatGPT one turns
+// max_output_tokens away) and another magpie, which fills its own, aren't.
+func fillsOutput(p provider.Provider, to provider.Protocol) bool {
+ if to == provider.Anthropic {
+ return true
+ }
+ return (to == provider.Chat || to == provider.Responses) &&
+ (p.Account == nil || p.IsPlugin()) && !p.IsRemoteMagpie()
+}
+
+// outputFieldRefused is a request error about the reply limit itself
+// ("max_tokens must be at most 32768"), not the window it shares with the
+// prompt, which replyLeft reads and the request is asked again for.
+func outputFieldRefused(status int, body []byte) bool {
+ if !badRequest(status) {
+ return false
+ }
+ w := wordsOf(body)
+ if replyLeft(w) > 0 {
+ return false
+ }
+ m := strings.ToLower(string(w))
+ return strings.Contains(m, "max_tokens") || strings.Contains(m, "max_output_tokens") ||
+ strings.Contains(m, "max_completion_tokens") || strings.Contains(m, "maxoutputtokens")
+}
+
+// outputFillRefused names, for markUnfit, a model whose vendor turned
+// away the reply limit magpie filled in: a request naming none is sent
+// none there from then on, as before.
+func outputFillRefused(model string) string { return "filled-output:" + model }
diff --git a/internal/gateway/own_output_test.go b/internal/gateway/own_output_test.go
new file mode 100644
index 000000000..20323a8ea
--- /dev/null
+++ b/internal/gateway/own_output_test.go
@@ -0,0 +1,205 @@
+package gateway
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/yetone/magpie/internal/catalog"
+ "github.com/yetone/magpie/internal/provider"
+)
+
+// ownOutputUpstream answers every API magpie may ask on, whole, and keeps
+// each body it was sent. A Chat body naming max_tokens past refuseOver is
+// turned away as a vendor whose limit is lower than its list says does.
+type ownOutputUpstream struct {
+ mu sync.Mutex
+ got []map[string]any
+ refuseOver int
+}
+
+func (u *ownOutputUpstream) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ b, _ := io.ReadAll(r.Body)
+ var q map[string]any
+ json.Unmarshal(b, &q)
+ u.mu.Lock()
+ u.got = append(u.got, q)
+ refuse := u.refuseOver
+ u.mu.Unlock()
+ stream := q["stream"] == true
+ if stream {
+ w.Header().Set("Content-Type", "text/event-stream")
+ } else {
+ w.Header().Set("Content-Type", "application/json")
+ }
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/responses") && stream:
+ io.WriteString(w, "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"r\",\"status\":\"in_progress\",\"output\":[]}}\n\n"+
+ "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"ok\"}\n\n"+
+ "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"r\",\"status\":\"completed\",\"output\":[{\"type\":\"message\",\"id\":\"o\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"ok\",\"annotations\":[]}]}],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n")
+ case strings.HasSuffix(r.URL.Path, "/responses"):
+ io.WriteString(w, `{"id":"r","object":"response","status":"completed","model":"m","output":[{"type":"message","id":"o","role":"assistant","status":"completed","content":[{"type":"output_text","text":"ok","annotations":[]}]}],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}`)
+ default:
+ if n, _ := q["max_tokens"].(float64); refuse > 0 && int(n) > refuse {
+ w.WriteHeader(400)
+ io.WriteString(w, `{"error":{"message":"Invalid max_tokens value, the valid range of max_tokens is [1, 32768]","type":"invalid_request_error"}}`)
+ return
+ }
+ if stream {
+ io.WriteString(w, "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\n"+
+ "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}\n\n"+
+ "data: [DONE]\n\n")
+ return
+ }
+ io.WriteString(w, `{"id":"c","object":"chat.completion","model":"m","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`)
+ }
+}
+
+func (u *ownOutputUpstream) take() []map[string]any {
+ u.mu.Lock()
+ defer u.mu.Unlock()
+ got := u.got
+ u.got = nil
+ return got
+}
+
+// Codex names no max_output_tokens, and DeepSeek's API reads none as 8K
+// without thinking, 64K with it, where its models give 384K: Codex's turns
+// on DeepSeek stopped there, "Incomplete response returned, reason:
+// max_output_tokens" (xuanwo on X). A request naming no reply limit is
+// sent the model's own there, relayed or translated, on every API a client
+// may speak; a limit the client named goes as it is, and the same request
+// to another vendor's Responses or Chat, relayed, goes as it came.
+func TestShortDefaultAsksTheModelsOutput(t *testing.T) {
+ fresh(t)
+ u := &ownOutputUpstream{}
+ up := httptest.NewServer(u)
+ t.Cleanup(up.Close)
+ os.MkdirAll(filepath.Dir(catalog.CachePath()), 0o755)
+ if err := os.WriteFile(catalog.CachePath(), []byte(`{"deepseek":{"id":"deepseek","models":{
+ "deepseek-flash":{"id":"deepseek-flash","limit":{"context":1048576,"output":393216}}}}}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ catalog.Reset()
+ t.Cleanup(catalog.Reset)
+ for _, p := range []provider.Provider{
+ {ID: "deepseek", Name: "DeepSeek", Key: "k", Catalog: "deepseek", Models: []string{"deepseek-flash"},
+ Chat: "http://api.deepseek.com/v1", Responses: "http://api.deepseek.com/v1"},
+ // the same model on a relay that speaks Responses and Chat: its
+ // default isn't DeepSeek's
+ {ID: "relay", Name: "Relay", Key: "k", Catalog: "deepseek", Models: []string{"deepseek-flash"},
+ Chat: "http://relay.example.com/v1", Responses: "http://relay.example.com/v1"},
+ } {
+ if err := provider.Save(p); err != nil {
+ t.Fatal(err)
+ }
+ }
+ s := New()
+ addr := strings.TrimPrefix(up.URL, "http://")
+ s.client = &http.Client{Transport: &http.Transport{DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) {
+ return (&net.Dialer{}).DialContext(ctx, network, addr)
+ }}}
+ const chat = `{"model":"%s","messages":[{"role":"user","content":"hi"}]}`
+ for _, c := range []struct {
+ name, path, body, field string
+ want int // 0: the field isn't sent
+ }{
+ {"Codex's turn, relayed", "/v1/responses", `{"model":"deepseek/deepseek-flash","input":"hi"}`, "max_output_tokens", 393216},
+ {"Codex's turn, its own limit", "/v1/responses", `{"model":"deepseek/deepseek-flash","input":"hi","max_output_tokens":1000}`, "max_output_tokens", 1000},
+ {"a Chat client, relayed", "/v1/chat/completions", strings.Replace(chat, "%s", "deepseek/deepseek-flash", 1), "max_tokens", 393216},
+ {"a Chat client, its own limit", "/v1/chat/completions", `{"model":"deepseek/deepseek-flash","max_completion_tokens":2000,"messages":[{"role":"user","content":"hi"}]}`, "max_tokens", 2000},
+ {"a Gemini client, translated", "/v1beta/models/deepseek/deepseek-flash:generateContent", `{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`, "", 393216},
+ // another host's Responses and Chat, relayed: as they came
+ {"a relay's Responses", "/v1/responses", `{"model":"relay/deepseek-flash","input":"hi"}`, "max_output_tokens", 0},
+ {"a relay's Chat", "/v1/chat/completions", strings.Replace(chat, "%s", "relay/deepseek-flash", 1), "max_tokens", 0},
+ } {
+ if code, raw := postJSON(t, s, c.path, c.body); code != 200 {
+ t.Fatalf("%s: %d %s", c.name, code, raw)
+ }
+ got := u.take()
+ if len(got) != 1 {
+ t.Fatalf("%s: %d requests upstream", c.name, len(got))
+ }
+ field := c.field
+ if field == "" {
+ // translated: whichever API magpie spoke
+ for _, k := range []string{"max_tokens", "max_completion_tokens", "max_output_tokens"} {
+ if _, ok := got[0][k]; ok {
+ field = k
+ }
+ }
+ }
+ n, sent := got[0][field].(float64)
+ if c.want == 0 && sent || c.want != 0 && int(n) != c.want {
+ t.Errorf("%s: %s %v (sent %v), want %d: %v", c.name, field, n, sent, c.want, got[0])
+ }
+ }
+}
+
+// Translated onto another vendor's Chat or Responses, a request naming no
+// reply limit is sent its model's own, as on Anthropic's (#1792): the
+// client's API reads none as the model's whole, which the vendor's needn't.
+// A vendor that turns the figure away (a list saying more than it takes) is
+// asked again naming none, as the client did, and from then on.
+func TestTranslatedAsksTheModelsOutput(t *testing.T) {
+ fresh(t)
+ u := &ownOutputUpstream{}
+ up := httptest.NewServer(u)
+ t.Cleanup(up.Close)
+ os.MkdirAll(filepath.Dir(catalog.CachePath()), 0o755)
+ if err := os.WriteFile(catalog.CachePath(), []byte(`{"kimi":{"id":"kimi","models":{
+ "k-big":{"id":"k-big","limit":{"context":262144,"output":65536}}}}}`), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ catalog.Reset()
+ t.Cleanup(catalog.Reset)
+ // a vendor serving Chat alone: Codex's turn is translated
+ if err := provider.Save(provider.Provider{ID: "v", Name: "V", Key: "k", Catalog: "kimi", Models: []string{"k-big"}, Chat: up.URL + "/v1"}); err != nil {
+ t.Fatal(err)
+ }
+ s := New()
+ codex := `{"model":"v/k-big","input":"hi"}`
+ ask := func() []map[string]any {
+ t.Helper()
+ if code, raw := postJSON(t, s, "/v1/responses", codex); code != 200 {
+ t.Fatalf("%d %s", code, raw)
+ }
+ return u.take()
+ }
+ got := ask()
+ if len(got) != 1 || got[0]["max_tokens"] != float64(65536) {
+ t.Fatalf("translated onto Chat: %v, want max_tokens 65536", got)
+ }
+ if code, _ := postJSON(t, s, "/v1/responses", `{"model":"v/k-big","input":"hi","max_output_tokens":700}`); code != 200 {
+ t.Fatal(code)
+ }
+ if got := u.take(); len(got) != 1 || got[0]["max_tokens"] != float64(700) {
+ t.Fatalf("the client's own limit: %v, want max_tokens 700", got)
+ }
+
+ u.mu.Lock()
+ u.refuseOver = 32768
+ u.mu.Unlock()
+ got = ask()
+ if len(got) != 2 || got[0]["max_tokens"] != float64(65536) {
+ t.Fatalf("refused: %v, want 65536 then none", got)
+ }
+ if _, ok := got[1]["max_tokens"]; ok {
+ t.Fatalf("asked again with a limit: %v", got[1])
+ }
+ got = ask()
+ if len(got) != 1 {
+ t.Fatalf("after a refusal: %d requests, want 1 naming none", len(got))
+ }
+ if _, ok := got[0]["max_tokens"]; ok {
+ t.Fatalf("filled in again after the vendor turned it away: %v", got[0])
+ }
+}