Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Send one flat max_tokens for Metabot chat, and retire the llm-max-tokens setting (#82436)

Paolo Rascunàbbf81b462c3d
diff --git a/docs/configuring-metabase/config-template.md b/docs/configuring-metabase/config-template.md
index 51c24fc997c..91b679050a7 100644
--- a/docs/configuring-metabase/config-template.md
+++ b/docs/configuring-metabase/config-template.md
@@ -227,7 +227,6 @@ config:
llm-google-oauth-access-token: null
llm-google-project-id: null
llm-google-service-account-key: null
- llm-max-tokens: 4096
llm-metabot-provider: anthropic/claude-sonnet-4-6
llm-mini-model: null
llm-mistral-api-base-url: https://api.mistral.ai/v1
diff --git a/docs/configuring-metabase/environment-variables.md b/docs/configuring-metabase/environment-variables.md
index c25b7eb20fb..c6ef18ce363 100644
--- a/docs/configuring-metabase/environment-variables.md
+++ b/docs/configuring-metabase/environment-variables.md
@@ -1523,14 +1523,6 @@ A Google Cloud service account key JSON for the Gemini Enterprise Agent Platform
Backed by the google connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.
-### `MB_LLM_MAX_TOKENS`
-
-- Type: integer
-- Default: `4096`
-- [Configuration file name](./config-file.md): `llm-max-tokens`
-
-Maximum tokens for LLM responses.
-
### `MB_LLM_METABOT_PROVIDER`
- Type: string
diff --git a/src/metabase/llm/settings.clj b/src/metabase/llm/settings.clj
index b785ecd2e77..2bbbd470817 100644
--- a/src/metabase/llm/settings.clj
+++ b/src/metabase/llm/settings.clj
@@ -446,13 +446,6 @@
:default true
:export? true)
-(defsetting llm-max-tokens
- (deferred-tru "Maximum tokens for LLM responses.")
- :type :integer
- :default 4096
- :visibility :settings-manager
- :export? false)
-
(defsetting llm-request-timeout-ms
(deferred-tru
(str "Socket (inter-byte read) timeout in milliseconds for LLM API requests. "
diff --git a/src/metabase/metabot/self/bedrock.clj b/src/metabase/metabot/self/bedrock.clj
index 99841f9b469..f3c1b57f965 100644
--- a/src/metabase/metabot/self/bedrock.clj
+++ b/src/metabase/metabot/self/bedrock.clj
@@ -418,6 +418,16 @@
[body]
(dissoc body :cache_control))
+(defn ->mantle-openai-body
+ "Adapt a canonical OpenAI Responses request body for the mantle endpoint.
+
+ A body without `max_output_tokens` gets [[core/chat-max-output-tokens]]."
+ [body]
+ ;; Unlike OpenAI direct, an unset cap costs quota here: mantle's quota check counts "the value of `max_tokens` (or
+ ;; the model-specific maximum if `max_tokens` is not set)"
+ ;; (https://docs.aws.amazon.com/bedrock/latest/userguide/quotas-mantle.html).
+ (update body :max_output_tokens #(or % core/chat-max-output-tokens)))
+
(mu/defn bedrock-raw
"Perform a streaming request to Bedrock: the mantle endpoint for a mantle catalog ID, `bedrock-runtime` for a
Claude model only it serves (see [[runtime-model?]]).
@@ -439,7 +449,7 @@
(claude/claude-request-body (assoc opts :model (runtime-base-model model))))
:read-stream runtime-events}
:openai {:path "/openai/v1/responses"
- :req (openai/openai-request-body opts)})]
+ :req (->mantle-openai-body (openai/openai-request-body opts))})]
(adapter/stream! (if (= :runtime family) runtime-provider provider) opts
{:path path
:body req
diff --git a/src/metabase/metabot/self/claude.clj b/src/metabase/metabot/self/claude.clj
index 15f232483e8..ea2d4101d51 100644
--- a/src/metabase/metabot/self/claude.clj
+++ b/src/metabase/metabot/self/claude.clj
@@ -347,24 +347,20 @@
(def supported-models
"Anthropic chat models offered in the Metabot model picker, keyed by model id.
`list-models` returns the intersection of this map with the account's `/v1/models` catalog."
- {"claude-fable-5-1" {:display-name "Claude Fable 5.1" :max-tokens 128000 :context-window 1000000}
- "claude-fable-5" {:display-name "Claude Fable 5" :max-tokens 128000 :context-window 1000000}
- "claude-opus-5-5" {:display-name "Claude Opus 5.5" :max-tokens 128000 :context-window 1000000}
- "claude-opus-5" {:display-name "Claude Opus 5" :max-tokens 128000 :context-window 1000000}
- "claude-opus-4-8" {:display-name "Claude Opus 4.8" :max-tokens 128000 :context-window 1000000}
- "claude-opus-4-7" {:display-name "Claude Opus 4.7" :max-tokens 128000 :context-window 1000000}
- "claude-opus-4-6" {:display-name "Claude Opus 4.6" :max-tokens 128000 :context-window 1000000}
- "claude-opus-4-5-20251101" {:display-name "Claude Opus 4.5" :max-tokens 64000 :context-window 200000}
- "claude-opus-4-1-20250805" {:display-name "Claude Opus 4.1" :max-tokens 32000 :context-window 200000}
- "claude-sonnet-5-5" {:display-name "Claude Sonnet 5.5" :max-tokens 128000 :context-window 1000000}
- "claude-sonnet-5" {:display-name "Claude Sonnet 5" :max-tokens 128000 :context-window 1000000}
- "claude-sonnet-4-6" {:display-name "Claude Sonnet 4.6" :max-tokens 128000 :context-window 1000000}
- "claude-sonnet-4-5-20250929" {:display-name "Claude Sonnet 4.5" :max-tokens 64000 :context-window 200000}
- "claude-haiku-4-5-20251001" {:display-name "Claude Haiku 4.5" :max-tokens 64000 :context-window 200000}})
-
-(def ^:private default-max-tokens
- "`max_tokens` for an unresolved model — low enough to be safe on any of them."
- 64000)
+ {"claude-fable-5-1" {:display-name "Claude Fable 5.1" :context-window 1000000}
+ "claude-fable-5" {:display-name "Claude Fable 5" :context-window 1000000}
+ "claude-opus-5-5" {:display-name "Claude Opus 5.5" :context-window 1000000}
+ "claude-opus-5" {:display-name "Claude Opus 5" :context-window 1000000}
+ "claude-opus-4-8" {:display-name "Claude Opus 4.8" :context-window 1000000}
+ "claude-opus-4-7" {:display-name "Claude Opus 4.7" :context-window 1000000}
+ "claude-opus-4-6" {:display-name "Claude Opus 4.6" :context-window 1000000}
+ "claude-opus-4-5-20251101" {:display-name "Claude Opus 4.5" :context-window 200000}
+ "claude-opus-4-1-20250805" {:display-name "Claude Opus 4.1" :context-window 200000}
+ "claude-sonnet-5-5" {:display-name "Claude Sonnet 5.5" :context-window 1000000}
+ "claude-sonnet-5" {:display-name "Claude Sonnet 5" :context-window 1000000}
+ "claude-sonnet-4-6" {:display-name "Claude Sonnet 4.6" :context-window 1000000}
+ "claude-sonnet-4-5-20250929" {:display-name "Claude Sonnet 4.5" :context-window 200000}
+ "claude-haiku-4-5-20251001" {:display-name "Claude Haiku 4.5" :context-window 200000}})
(mu/defn list-models :- adapter/ModelListing
"List the Anthropic chat models supported by this adapter, by intersecting [[supported-models]] with the
@@ -384,11 +380,6 @@
[model]
(str/replace-first (u/lower-case-en (str model)) #"^anthropic\." ""))
-(defn- model-max-tokens
- "The `max_tokens` ceiling for `model`, or nil when it isn't one we know."
- [model]
- (get-in supported-models [(strip-vendor-prefix model) :max-tokens]))
-
(mu/defn context-window-tokens :- [:maybe :int]
"The input context window for `model`, or nil when it isn't one we know."
[model :- [:maybe :string]]
@@ -496,7 +487,8 @@
(add-tools-cache-breakpoint all-tools)
all-tools)]
(cond-> {:model model
- :max_tokens (or max-tokens (model-max-tokens model) default-max-tokens)
+ ;; required by the Messages API (https://platform.claude.com/docs/en/api/messages), so never omitted
+ :max_tokens (or max-tokens core/chat-max-output-tokens)
:stream true
:cache_control {:type "ephemeral"}
:messages messages}
diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.clj
index 0065a527983..f55347e8b99 100644
--- a/src/metabase/metabot/self/core.clj
+++ b/src/metabase/metabot/self/core.clj
@@ -274,6 +274,26 @@
[:fast? {:optional true} [:maybe :boolean]]
[:prompt-cache-key {:optional true} [:maybe :string]]])
+(def chat-max-output-tokens
+ "The output-token cap for Metabot chat, sent when the caller passes no `:max-tokens` — only the agent loop does.
+
+ Sized for Metabot's chat rather than for any model: production chat output has a p99.9 of about 7,300 tokens, so
+ 32000 truncates only a runaway generation, and it is at or below every catalog model's documented maximum, the
+ lowest being Claude Opus 4.1's 32,000
+ (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-4-1.html).
+
+ A constant rather than a setting: the output distribution is the same on every instance, telemetry records no
+ finish reason so a lowered value would truncate tool calls invisibly, and some surfaces deliberately send no cap
+ (see the OpenAI builder, and Google Model Garden endpoints, which reach the vLLM builder with no context window)
+ or send it only when it fits the context window (see the vLLM builder), so one global knob would mislead.
+
+ Where a provider counts the cap against the context window, the cap also takes room from the prompt. Mistral
+ (https://docs.mistral.ai/api/endpoint/chat) and Moonshot (https://platform.kimi.ai/docs/api/chat) reject a
+ request whose prompt plus cap exceeds the window, and both send this cap on every request: on a 262,144-token
+ window a prompt over 230,144 tokens fails while the context meter shows about 88%. The agent loop resends the
+ full history and does not compact it, so a long enough conversation reaches this limit."
+ 32000)
+
(defn mkid
"Generate a random id"
[]
diff --git a/src/metabase/metabot/self/google/raw_predict.clj b/src/metabase/metabot/self/google/raw_predict.clj
index b5853bfa60e..6da7cf08149 100644
--- a/src/metabase/metabot/self/google/raw_predict.clj
+++ b/src/metabase/metabot/self/google/raw_predict.clj
@@ -23,8 +23,8 @@
(defn- direct-api-model-id
"The direct-API spelling of a platform model ID.
The Gemini Agent Platform versions a dated model as `{model}@{date}` where the Messages API spells it
- `{model}-{date}`. The Claude adapter's model knowledge (max_tokens ceilings, thinking support) is keyed by the
- direct spelling."
+ `{model}-{date}`. The Claude adapter's model knowledge (thinking support, context windows) is keyed by the direct
+ spelling."
[model-id]
(str/replace (str model-id) "@" "-"))
diff --git a/src/metabase/metabot/self/google/stream_generate_content.clj b/src/metabase/metabot/self/google/stream_generate_content.clj
index 335416d4049..a2f78a0a0a3 100644
--- a/src/metabase/metabot/self/google/stream_generate_content.clj
+++ b/src/metabase/metabot/self/google/stream_generate_content.clj
@@ -158,7 +158,9 @@
2048)
(mu/defn request-body
- "Builds the `streamGenerateContent` request body for an LLM request."
+ "Builds the `streamGenerateContent` request body for an LLM request.
+
+ A request that carries no `:max-tokens` is capped at [[core/chat-max-output-tokens]]."
[{:keys [system input tools schema tool_choice temperature max-tokens model reasoning?]
:or {reasoning? true}} :- core/LLMRequestOpts]
(let [fdecls (when (seq tools) (mapv tool->function-declaration tools))
@@ -181,13 +183,14 @@
;; The chat path streams to the browser: ask for the thought summaries the
;; chain-of-thought UI renders, and leave the default thinking level alone.
reasoning? {:includeThoughts true}))
+ max-tokens (or max-tokens core/chat-max-output-tokens)
;; Safety net: the forced tool call must survive the un-disableable thinking spend, which
- ;; Gemini bills against maxOutputTokens (see [[forced-tool-call-token-floor]]). Only an
- ;; existing cap is raised, and only where a tool call is actually forced — the chat path
- ;; sends no cap at all. Independent of :reasoning?, because a catalog model thinks whether
- ;; or not we asked it to.
+ ;; Gemini bills against maxOutputTokens (see [[forced-tool-call-token-floor]]). The cap is
+ ;; raised only where a tool call is actually forced. The default cap is well above the
+ ;; floor, so this bites only on a smaller caller cap. Independent of :reasoning?, because a
+ ;; catalog model thinks whether or not we asked it to.
max-tokens (cond-> max-tokens
- (and max-tokens forced? (models/reasoning-model? model))
+ (and forced? (models/reasoning-model? model))
(max forced-tool-call-token-floor))
gen-config (cond-> {}
max-tokens (assoc :maxOutputTokens max-tokens)
diff --git a/src/metabase/metabot/self/mistral.clj b/src/metabase/metabot/self/mistral.clj
index e5436d7ebdd..90c25426a72 100644
--- a/src/metabase/metabot/self/mistral.clj
+++ b/src/metabase/metabot/self/mistral.clj
@@ -101,8 +101,10 @@
- Prompt caching is opt-in per request via `prompt_cache_key` (cache reads bill at 10% of the input price), so a
`:prompt-cache-key` — the conversation id — is forwarded when present.
- Whitelisted models get a `reasoning_effort` directive and replay their in-turn reasoning as think chunks
- (see [[think-message]])."
- [{:keys [model prompt-cache-key reasoning? schema] :as opts
+ (see [[think-message]]).
+
+ A caller that names no `:max-tokens` gets [[core/chat-max-output-tokens]]."
+ [{:keys [model prompt-cache-key reasoning? schema max-tokens] :as opts
:or {model default-model reasoning? true}} :- core/LLMRequestOpts]
;; mistral-medium-3-5 accepts exactly "high" and "none" — the server 400s the other four
;; enum values, enumerating these two (probed 2026-09-01) — and its server default sends NO
@@ -117,7 +119,7 @@
;; from the replayed input, honoring the LLMRequestOpts contract.
thinking? (and whitelisted? reasoning? (not schema))]
(-> (chat-completions/request-body
- (assoc opts :model model)
+ (assoc opts :model model :max-tokens (or max-tokens core/chat-max-output-tokens))
(when thinking? {:reasoning-part->message think-message}))
(dissoc :stream_options)
(cond-> prompt-cache-key (assoc :prompt_cache_key prompt-cache-key)
diff --git a/src/metabase/metabot/self/moonshot.clj b/src/metabase/metabot/self/moonshot.clj
index c9177e50b0b..904f5c53f71 100644
--- a/src/metabase/metabot/self/moonshot.clj
+++ b/src/metabase/metabot/self/moonshot.clj
@@ -91,9 +91,9 @@
2048)
(defn- floor-max-tokens
- "Raises an existing `max_tokens` cap to [[forced-tool-call-token-floor]]; an uncapped body stays uncapped."
+ "Raises the `max_tokens` cap to [[forced-tool-call-token-floor]]."
[body]
- (cond-> body (:max_tokens body) (update :max_tokens max forced-tool-call-token-floor)))
+ (update body :max_tokens max forced-tool-call-token-floor))
(defn- reasoning-message
"The replayed assistant message for a coalesced in-turn :reasoning part.
@@ -125,8 +125,10 @@
otherwise; their in-turn reasoning is replayed (see [[reasoning-message]]) and forced tool calls get a
`max_tokens` floor (see [[forced-tool-call-token-floor]]).
- **`prompt_cache_key`.** Moonshot caching is automatic and hits without it, but a `:prompt-cache-key` — the
- conversation id — is forwarded when present."
- [{:keys [model prompt-cache-key reasoning? schema tool_choice] :as opts
+ conversation id — is forwarded when present.
+
+ A caller that names no `:max-tokens` gets [[core/chat-max-output-tokens]]."
+ [{:keys [model prompt-cache-key reasoning? schema tool_choice max-tokens] :as opts
:or {model default-model reasoning? true}} :- core/LLMRequestOpts]
;; kimi-k3 always thinks — there is no off switch, so `reasoning_effort` ("low" | "high" | "max", server default
;; "max") is the only knob (https://platform.kimi.ai/docs/guide/kimi-k3-quickstart). "max" on the chat path pins
@@ -148,7 +150,7 @@
thinking? (and whitelisted? reasoning?
(if thinking-only? (not schema) (not forced?)))]
(-> (chat-completions/request-body
- (assoc opts :model model)
+ (assoc opts :model model :max-tokens (or max-tokens core/chat-max-output-tokens))
;; Replay only where the dialect mandates it: k3's Preserved Thinking. k2.6 cannot use
;; `thinking.keep "all"` — that mode obliges the caller to send back EVERY historical assistant
;; message's reasoning_content, but reasoning parts are never persisted across turns, so we could
diff --git a/src/metabase/metabot/self/openai.clj b/src/metabase/metabot/self/openai.clj
index b2564e309c3..50cb66756a0 100644
--- a/src/metabase/metabot/self/openai.clj
+++ b/src/metabase/metabot/self/openai.clj
@@ -316,7 +316,9 @@
(reasoning-model? model))
(mu/defn openai-request-body
- "Build the OpenAI Responses API request body for an LLM request."
+ "Build the OpenAI Responses API request body for an LLM request.
+
+ `max_output_tokens` is sent only when the caller passes `:max-tokens`."
[{:keys [model system input tools schema tool_choice temperature max-tokens reasoning?]
:or {model default-model reasoning? true}} :- core/LLMRequestOpts]
(let [input (cond->> input
@@ -338,6 +340,11 @@
tool_choice tool_choice
:else "auto")
:tools all-tools)
+ ;; No default cap: OpenAI and Azure meter a sent cap against the rate limit — "Your rate limit is calculated
+ ;; as the maximum of max_tokens and the estimated number of tokens"
+ ;; (https://developers.openai.com/api/docs/guides/rate-limits,
+ ;; https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/quota). Both pages name max_tokens; nothing
+ ;; says the Responses field max_output_tokens is metered differently.
max-tokens (assoc :max_output_tokens max-tokens)
;; encrypted_content lets us replay reasoning items across tool-call
diff --git a/src/metabase/metabot/self/openrouter.clj b/src/metabase/metabot/self/openrouter.clj
index 2504ade2d72..83728f9d7f9 100644
--- a/src/metabase/metabot/self/openrouter.clj
+++ b/src/metabase/metabot/self/openrouter.clj
@@ -244,10 +244,10 @@
[body {:keys [model reasoning? schema] :or {reasoning? true}}]
(let [forced? (or (some? schema) (= "required" (:tool_choice body)))
;; Safety net: the mandatory tool call must survive the un-disableable thinking spend
- ;; (theory vs practice in [[forced-tool-call-token-floor]]); only an existing cap is
- ;; raised, and only where a tool call is actually forced.
+ ;; (theory vs practice in [[forced-tool-call-token-floor]]); the cap is raised only
+ ;; where a tool call is actually forced.
body (cond-> body
- (and (reasoning-mandatory? model) forced? (:max_tokens body))
+ (and (reasoning-mandatory? model) forced?)
(update :max_tokens max forced-tool-call-token-floor))]
(if-not (= :renderable (reasoning-class model))
body
@@ -274,10 +274,17 @@
`:temperature` is dropped for models that reject it (see [[model-supports-temperature?]]). Gating it in the shared
builder instead would apply these OpenRouter-specific rules to every Chat Completions adapter, including vLLM,
- whose model names are customer-chosen free text."
- [{:keys [model system] :as opts
+ whose model names are customer-chosen free text.
+
+ A caller that names no `:max-tokens` gets [[core/chat-max-output-tokens]]."
+ [{:keys [model system max-tokens] :as opts
:or {model default-model}} :- core/LLMRequestOpts]
- (-> (cond-> (chat-completions/request-body (assoc opts :model model))
+ ;; `openai/*` models get the default too: the rate-limit metering that keeps
+ ;; [[metabase.metabot.self.openai/openai-request-body]] from sending one is OpenAI's and Azure's, and uncapped,
+ ;; OpenRouter substitutes a per-endpoint default (probed 2026-09-17 on deepseek/deepseek-v4-pro: 16384 on one
+ ;; endpoint, 32768 on another).
+ (-> (cond-> (chat-completions/request-body
+ (assoc opts :model model :max-tokens (or max-tokens core/chat-max-output-tokens)))
(and system (anthropic-model? model))
(update-in [:messages 0 :content] claude/system->cached-content-blocks)
diff --git a/src/metabase/metabot/self/vllm.clj b/src/metabase/metabot/self/vllm.clj
index 8130fc9a5ac..7663dd38482 100644
--- a/src/metabase/metabot/self/vllm.clj
+++ b/src/metabase/metabot/self/vllm.clj
@@ -15,10 +15,12 @@
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
+ [metabase.util.log :as log]
[metabase.util.malli :as mu])
(:import
(java.io IOException)
(java.net SocketTimeoutException)
+ (java.nio.charset StandardCharsets)
(java.util.concurrent ExecutionException)))
(set! *warn-on-reflection* true)
@@ -175,17 +177,44 @@
2048)
(def ^:private forced-tool-call-token-floor
- "Smallest `max_tokens` a forced tool call is given, regardless of what the caller asked for — below
- it a reasoning model spends the budget thinking and emits no tool call. Equal to
- [[probe-max-tokens]], which [[preflight!]] already proves the served model can clear."
+ "Smallest `max_tokens` a cap on a forced tool call is raised to — below it a reasoning model spends
+ the budget thinking and emits no tool call. Equal to [[probe-max-tokens]], which [[preflight!]]
+ already proves the served model can clear."
probe-max-tokens)
(def ^:private reasoning-model-token-floor
- "Smallest `max_tokens` any request gets once [[preflight!]] has observed the served model reasoning.
- Chat Completions bills thinking, answer, and tool call against one budget. Mirrors
- `claude-request-body`."
+ "Smallest `max_tokens` a cap is raised to once [[preflight!]] has observed the served model
+ reasoning. Chat Completions bills thinking, answer, and tool call against one budget."
16384)
+(def ^:private prompt-overhead-tokens
+ "Tokens [[estimated-prompt-tokens]] adds for what the chat template wraps around the messages and
+ tools: role markers, a tool-use preamble, the generation prompt."
+ 1024)
+
+(def ^:private answered-lookup-ttl-ms
+ "How long [[served-max-model-len]] reuses the answer of a `/v1/models` lookup that succeeded, with or
+ without a window for the model.
+
+ Five minutes, because a vLLM server is seldom restarted after its initial setup. The cost: after a
+ restart with a smaller `--max-model-len`, a cap sized for the old window can get a 400 from vLLM for up
+ to five minutes after the lookup that cached it. A server that lists no window (Ollama, LM Studio) does
+ not start to list one, so it is asked again only at this interval too."
+ (* 5 60 1000))
+
+(def ^:private failed-lookup-ttl-ms
+ "How long [[served-max-model-len]] reuses the nil window of a `/v1/models` lookup that failed: a
+ timeout, a refused connection, or an HTTP error status.
+
+ Shorter than [[answered-lookup-ttl-ms]], so a server that comes up is used soon. A `/v1/models` that is
+ down still costs at most one [[window-lookup-timeout-ms]] lookup in this period."
+ 30000)
+
+(def ^:private window-lookup-timeout-ms
+ "Connection and socket timeout of the `/v1/models` lookup behind [[served-max-model-len]]. The
+ catalog is cheap to serve, and the chat request waits for this lookup."
+ 5000)
+
(def ^:private default-temperature
"Sampling temperature for a caller that supplies none. vLLM's own default is 1.0, which is wrong for
the tool-calling and SQL-generation work the agent loop does. The hosted providers pick a sane
@@ -395,22 +424,139 @@
;;; --------------------------------------------------- Requests -------------------------------------------------
+(defonce ^:private window-cache
+ ;; [base-url model] -> {:window n-or-nil, :expires-at epoch-ms}. Each entry carries its own expiry, because a
+ ;; answered and a failed lookup live for different times (see [[lookup-entry]]). The keys are the servers
+ ;; and models the connections name, so the map stays small without eviction.
+ (atom {}))
+
+(defn- now-ms
+ "The current time in epoch milliseconds. A function, so a test can set the clock."
+ []
+ (System/currentTimeMillis))
+
+(defn- window-entry
+ "A [[window-cache]] entry for `window` that expires `ttl-ms` from now."
+ [window ttl-ms]
+ {:window window
+ :expires-at (+ (now-ms) ttl-ms)})
+
+(defn- live-entry
+ "The [[window-cache]] entry for `k`, or nil when there is none or it has expired."
+ [k]
+ (let [entry (get @window-cache k)]
+ (when (and entry (< (now-ms) (:expires-at entry)))
+ entry)))
+
+(defn- fetch-max-model-len
+ "Fetch the context window the server at `credentials` advertises for `model`, or nil.
+
+ vLLM's `/v1/models` entries carry `max_model_len`
+ (https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/serve/engine/protocol.py, `ModelCard`).
+ Nil when the catalog lists no entry with the model's id or no window for it (Ollama, LM Studio, TGI).
+ Throws when the request fails."
+ [credentials model]
+ (let [res (adapter/request! provider
+ {:credentials credentials :method :get :path "/models" :as :json}
+ {:socket-timeout window-lookup-timeout-ms
+ :connection-timeout window-lookup-timeout-ms})
+ entry (u/seek #(= model (:id %)) (get-in res [:body :data]))]
+ (when (pos-int? (:max_model_len entry))
+ (:max_model_len entry))))
+
+(defn- lookup-entry
+ "Look up the window of `model` on the server at `credentials`, as a [[window-cache]] entry.
+
+ An answer, with or without a window, lives for [[answered-lookup-ttl-ms]]. A failed lookup gives a nil
+ window that lives for [[failed-lookup-ttl-ms]]: an unknown window only means no default cap, and the
+ chat request that follows reports a real failure in its own words."
+ [credentials model]
+ (try
+ (window-entry (fetch-max-model-len credentials model) answered-lookup-ttl-ms)
+ (catch Exception e
+ (log/debugf e "Could not read the context window of %s from the vLLM server" model)
+ (window-entry nil failed-lookup-ttl-ms))))
+
+(defn- served-max-model-len
+ "The context window the vLLM server serves the request's model with, or nil when it is unknown.
+
+ Read from the server rather than stored on the connection, so a restart with a different
+ `--max-model-len` is followed within [[answered-lookup-ttl-ms]]. Lookups are cached per base URL and
+ model, nil windows and failures too, so a server costs one lookup per [[answered-lookup-ttl-ms]], or per
+ [[failed-lookup-ttl-ms]] while its lookup fails, not one per request. Two requests that miss at the same
+ time both look up; the later answer wins."
+ [{:keys [model credentials ai-proxy?]}]
+ (let [base-url (:base-url credentials)
+ k [base-url model]]
+ (when-not (or ai-proxy? (str/blank? base-url))
+ (:window (or (live-entry k)
+ (let [entry (lookup-entry credentials model)]
+ (swap! window-cache assoc k entry)
+ entry))))))
+
+(defn- estimated-prompt-tokens
+ "A high estimate of the prompt tokens in the Chat Completions `body`.
+
+ The UTF-8 bytes of its JSON `messages` and `tools`, plus [[prompt-overhead-tokens]]. One byte counts as
+ one token: the server can host any model, and no token of a byte-level BPE or byte-fallback tokenizer
+ is shorter than one byte. That is an upper bound, not a typical rate. Metabot prose measures 4 to 5
+ bytes per token, but rows of numbers measure 1.0 on Qwen3, which gives each digit its own token, and
+ below 2 on a tokenizer that groups up to three digits. A high estimate is the safe side:
+ [[output-cap]] then sends no cap, where a low one would send a cap that does not fit."
+ [body]
+ (+ prompt-overhead-tokens
+ (alength (.getBytes ^String (json/encode (select-keys body [:messages :tools]))
+ StandardCharsets/UTF_8))))
+
+(defn- raise-to-floors
+ "Raise `cap` to [[forced-tool-call-token-floor]] on a `forced?` call, and to
+ [[reasoning-model-token-floor]] on a reasoning connection."
+ [cap forced? credentials]
+ (cond-> cap
+ forced? (max forced-tool-call-token-floor)
+ (reasoning-connection? credentials) (max reasoning-model-token-floor)))
+
+(defn- output-cap
+ "The `max_tokens` to send with `body`, or nil to send none.
+
+ For example, on a 131072-token window the agent loop's 48 KB first prompt gets 32000; on a 32768-token
+ window it gets none, and vLLM stops at the remaining window, about 21000 tokens.
+
+ The cap is the caller's `max-tokens`, or [[core/chat-max-output-tokens]] when the `window` is known,
+ raised by [[raise-to-floors]]. It is sent only when [[estimated-prompt-tokens]] plus the cap fits the
+ window: vLLM subtracts a sent cap from the admissible prompt and returns a 400 when the prompt no longer
+ fits (https://github.com/vllm-project/vllm/blob/main/vllm/renderers/params.py). A cap that does not fit
+ is dropped, not lowered. Uncapped, vLLM generates up to the remaining window, which then bounds the
+ output by the exact prompt length rather than by an estimate.
+
+ With an unknown window, only a caller's cap is sent."
+ [body max-tokens forced? credentials window]
+ (let [cap (some-> (or max-tokens (when window core/chat-max-output-tokens))
+ (raise-to-floors forced? credentials))]
+ (when (and cap
+ (or (nil? window)
+ (<= (+ (estimated-prompt-tokens body) cap) window)))
+ cap)))
+
(mu/defn vllm-request-body
"Build the Chat Completions request body for an LLM request.
- Matches what [[chat-completions/request-body]] emits, except that `max_tokens` is always sent —
- without a ceiling vLLM falls back to the remaining context window, so one looping small model
- consumes the whole budget in a single call — and is raised to
- [[forced-tool-call-token-floor]] or [[reasoning-model-token-floor]] where either applies, and
- `temperature` falls back to [[default-temperature]]. All three stay adapter-local rather than
- moving into the shared builder, which would also change Z.AI, Mistral, and OpenRouter."
- [{:keys [max-tokens temperature schema tool_choice credentials] :as opts} :- core/LLMRequestOpts]
- (let [forced? (or (some? schema) (= "required" (some-> tool_choice name)))]
- (assoc (chat-completions/request-body (cond-> opts
- (nil? temperature) (assoc :temperature default-temperature)))
- :max_tokens (cond-> (or max-tokens (llm/llm-max-tokens))
- forced? (max forced-tool-call-token-floor)
- (reasoning-connection? credentials) (max reasoning-model-token-floor)))))
+ Matches what [[chat-completions/request-body]] emits, except that `max_tokens` is the one
+ [[output-cap]] picks for the context window `max-model-len` (nil when unknown), and `temperature`
+ falls back to [[default-temperature]]. Both stay adapter-local rather than moving into the shared
+ builder, which would also change Z.AI, Mistral, and OpenRouter.
+
+ Pure: [[vllm-raw]] looks the window up. The 1-arity, which Model Garden endpoints use, has none."
+ ([opts :- core/LLMRequestOpts]
+ (vllm-request-body opts nil))
+ ([{:keys [max-tokens temperature schema tool_choice credentials] :as opts} :- core/LLMRequestOpts
+ max-model-len :- [:maybe pos-int?]]
+ (let [forced? (or (some? schema) (= "required" (some-> tool_choice name)))
+ body (chat-completions/request-body (cond-> (dissoc opts :max-tokens)
+ (nil? temperature) (assoc :temperature default-temperature)))
+ cap (output-cap body max-tokens forced? credentials max-model-len)]
+ (cond-> body
+ cap (assoc :max_tokens cap)))))
(defn- stream-io-ex
"The vLLM error for a transport failure while *consuming* a response stream. Tagged
@@ -467,14 +613,15 @@
"Perform a streaming request to a vLLM server's Chat Completions API.
Opts map takes `:credentials` (`{:base-url ... :api-key ...}`) from the connection serving this
request, and throws without a base URL.
- `:ai-proxy?` is not supported for vLLM and throws when true."
+ `:ai-proxy?` is not supported for vLLM and throws when true.
+ Reads the served context window first, through [[served-max-model-len]]'s cache, to size `max_tokens`."
[{:keys [model credentials] :as opts} :- core/LLMRequestOpts]
(when (str/blank? model)
(throw (missing-model-ex)))
(let [timeout-ms (llm/llm-vllm-request-timeout-ms)]
(adapter/stream! provider opts
{:path "/chat/completions"
- :body (vllm-request-body opts)
+ :body (vllm-request-body opts (served-max-model-len opts))
:request-options (inference-timeouts)
:wrap-stream #(io-guarded % timeout-ms)
;; clj-http raises an `IOException` only when there is no response at all, so the
diff --git a/src/metabase/metabot/self/xai.clj b/src/metabase/metabot/self/xai.clj
index c6233324f8d..7a20c554592 100644
--- a/src/metabase/metabot/self/xai.clj
+++ b/src/metabase/metabot/self/xai.clj
@@ -61,17 +61,20 @@
xAI's dialect matches what [[chat-completions/request-body]] emits, except:
- The output cap is sent as `max_completion_tokens`, since xAI deprecates `max_tokens`. It caps visible output
- only, not reasoning or tool calls, so a small cap cannot cut off a forced tool call.
+ only, not reasoning or tool calls, so a small cap cannot cut off a forced tool call. A caller that names no
+ `:max-tokens` gets [[core/chat-max-output-tokens]], since xAI's own default is 128,000 visible tokens
+ (https://docs.x.ai/developers/rest-api-reference/inference/chat-completions).
- The structured path, and a caller opting out of reasoning, get the model's `:lowest-effort` from
[[supported-models]], which turns reasoning off on Grok 4.3 and lowers it to `low` on Grok 4.7. Chat keeps each
model's default. Models off the allow-list get no effort at all, since some of them (Grok 4.20) reject the
parameter.
- A `:prompt-cache-key`, the conversation id, is forwarded as `prompt_cache_key`, which routes a conversation's
requests to the same server so its prompt cache hits."
- [{:keys [model prompt-cache-key reasoning? schema] :as opts
+ [{:keys [model max-tokens prompt-cache-key reasoning? schema] :as opts
:or {model default-model reasoning? true}} :- core/LLMRequestOpts]
(let [lowest-effort (get-in supported-models [model :lowest-effort])]
- (-> (chat-completions/request-body (assoc opts :model model))
+ (-> (chat-completions/request-body
+ (assoc opts :model model :max-tokens (or max-tokens core/chat-max-output-tokens)))
(set/rename-keys {:max_tokens :max_completion_tokens})
(cond-> (and lowest-effort (or (some? schema) (not reasoning?)))
(assoc :reasoning_effort lowest-effort)
diff --git a/src/metabase/metabot/self/zai.clj b/src/metabase/metabot/self/zai.clj
index 887a69b705d..b866d910ca1 100644
--- a/src/metabase/metabot/self/zai.clj
+++ b/src/metabase/metabot/self/zai.clj
@@ -91,9 +91,10 @@
adding Z.AI's `thinking` directive: enabled only where a whitelisted model's reasoning renders, disabled
otherwise. A [[thinking-only-model?]] rejects the directive and gets `reasoning_effort` instead — \"max\"
where reasoning renders, \"low\" otherwise — plus a `max_tokens` floor on forced tool calls (see
- [[forced-tool-call-token-floor]]). Z.AI documents only `tool_choice \"auto\"`, but `\"required\"` — which the
- structured-output path relies on — is accepted and honored in practice, with thinking on."
- [{:keys [model reasoning? schema tool_choice] :as opts
+ [[forced-tool-call-token-floor]]). A caller that passes no cap of its own gets [[core/chat-max-output-tokens]]. Z.AI
+ documents only `tool_choice \"auto\"`, but `\"required\"` — which the structured-output path relies on — is
+ accepted and honored in practice, with thinking on."
+ [{:keys [model reasoning? schema tool_choice max-tokens] :as opts
:or {model default-model reasoning? true}} :- core/LLMRequestOpts]
;; Thinking is on by default server-side, at reasoning_effort "max"
;; (https://docs.z.ai/api-reference/llm/chat-completion), so "enabled" only makes the default
@@ -109,12 +110,13 @@
(let [thinking-only? (thinking-only-model? model)
forced? (or (some? schema) (= "required" (some-> tool_choice name)))
thinking? (and (reasoning-model? model) reasoning? (not schema))
- body (chat-completions/request-body (assoc opts :model model))]
+ max-tokens (or max-tokens core/chat-max-output-tokens)
+ body (chat-completions/request-body (assoc opts :model model :max-tokens max-tokens))]
(cond-> body
thinking-only?
(assoc :reasoning_effort (if thinking? "max" "low"))
- (and thinking-only? forced? (:max_tokens body))
+ (and thinking-only? forced?)
(update :max_tokens max forced-tool-call-token-floor)
(not thinking-only?)
diff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.clj
index 9ad210d73d9..30c18eaa451 100644
--- a/src/metabase/settings/models/setting.clj
+++ b/src/metabase/settings/models/setting.clj
@@ -77,7 +77,8 @@
"metabot-enabled"
"ldap-sync-admin-group"
"user-recent-views"
- "most-recently-viewed-dashboard"})
+ "most-recently-viewed-dashboard"
+ "llm-max-tokens"})
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:dynamic *allow-retired-setting-names*
diff --git a/test/metabase/llm/settings_test.clj b/test/metabase/llm/settings_test.clj
index 9115f9a1689..1995e65aaac 100644
--- a/test/metabase/llm/settings_test.clj
+++ b/test/metabase/llm/settings_test.clj
@@ -397,14 +397,6 @@
;;; ------------------------------------------- Settings Defaults Tests -------------------------------------------
-(deftest llm-max-tokens-test
- (testing "default value is 4096"
- (mt/with-temporary-setting-values [llm-max-tokens nil]
- (is (= 4096 (llm.settings/llm-max-tokens)))))
- (testing "can be overridden"
- (mt/with-temporary-setting-values [llm-max-tokens 8192]
- (is (= 8192 (llm.settings/llm-max-tokens))))))
-
(deftest llm-request-timeout-ms-test
(testing "default value is 120000 (120 seconds)"
(mt/with-temporary-setting-values [llm-request-timeout-ms nil]
diff --git a/test/metabase/metabot/self/azure_test.clj b/test/metabase/metabot/self/azure_test.clj
index 1ff0c21862f..789e59398c7 100644
--- a/test/metabase/metabot/self/azure_test.clj
+++ b/test/metabase/metabot/self/azure_test.clj
@@ -135,8 +135,8 @@
:system [{:type "text" :text "be brief" :cache_control {:type "ephemeral"}}]
:messages [{:role "user" :content [{:type "text" :text "hi"}]}]}
body)))
- (testing "a deployment name matches no model, so max_tokens falls back rather than being omitted"
- (is (= 64000 (:max_tokens body))))))
+ (testing "a deployment gets the default max_tokens rather than none"
+ (is (= 32000 (:max_tokens body))))))
(deftest openai-family-dispatches-to-responses-api-test
(let [req (captured-raw-request! {:model "openai/gpt-5-deployment"
diff --git a/test/metabase/metabot/self/bedrock_test.clj b/test/metabase/metabot/self/bedrock_test.clj
index 4caae8912f5..858b240dc4a 100644
--- a/test/metabase/metabot/self/bedrock_test.clj
+++ b/test/metabase/metabot/self/bedrock_test.clj
@@ -322,12 +322,12 @@
(json/decode+kw (:body (captured-raw-request! (merge {:input [{:role :user :content "hi"}]} opts)))))
(deftest anthropic-model-max-tokens-test
- (testing "the `anthropic.` prefix is stripped so the model's own ceiling resolves"
+ (testing "anthropic.* models get the default cap, and the caller's own cap wins"
(are [opts tokens] (= tokens (:max_tokens (captured-body! opts)))
- {:model "anthropic.claude-opus-4-8"} 128000
+ {:model "anthropic.claude-opus-4-8"} 32000
{:model "anthropic.claude-opus-4-8" :max-tokens 128} 128))
- (testing "openai.* models omit the field entirely"
- (is (not (contains? (captured-body! {:model "openai.gpt-5.5"}) :max_output_tokens)))))
+ (testing "openai.* models get the default cap too, unlike on OpenAI direct"
+ (is (= 32000 (:max_output_tokens (captured-body! {:model "openai.gpt-5.5"}))))))
(deftest reasoning-request-config-test
(testing "anthropic models request adaptive summarized thinking, and only that"
@@ -416,6 +416,14 @@
:cache_control {:type "ephemeral"}
:system [{:type "text" :text "s" :cache_control {:type "ephemeral"}}]})))))
+(deftest ^:parallel mantle-openai-body-test
+ (testing "adds the default max_output_tokens where the body has none"
+ (is (= {:model "openai.gpt-5.5" :max_output_tokens 32000}
+ (bedrock/->mantle-openai-body {:model "openai.gpt-5.5"}))))
+ (testing "keeps a caller's own cap"
+ (is (= {:model "openai.gpt-5.5" :max_output_tokens 512}
+ (bedrock/->mantle-openai-body {:model "openai.gpt-5.5" :max_output_tokens 512})))))
+
;;; ──────────────────────────────────────────────────────────────────
;;; Stream translation (xf selection by model family)
;;; ──────────────────────────────────────────────────────────────────
diff --git a/test/metabase/metabot/self/chat_max_tokens_test.clj b/test/metabase/metabot/self/chat_max_tokens_test.clj
new file mode 100644
index 00000000000..61262a5cba5
--- /dev/null
+++ b/test/metabase/metabot/self/chat_max_tokens_test.clj
@@ -0,0 +1,294 @@
+(ns metabase.metabot.self.chat-max-tokens-test
+ "The output-token cap every surface sends when the caller names none.
+
+ One namespace across every adapter, because the property is cross-provider: every surface sends the same default
+ cap, whatever the model and however its provider spells the id, except the few that deliberately send none."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase.llm.provider :as llm.provider]
+ [metabase.metabot.self.azure :as azure]
+ [metabase.metabot.self.bedrock :as bedrock]
+ [metabase.metabot.self.claude :as claude]
+ [metabase.metabot.self.deepseek :as deepseek]
+ [metabase.metabot.self.google.models :as google.models]
+ [metabase.metabot.self.google.raw-predict :as raw-predict]
+ [metabase.metabot.self.google.stream-generate-content :as stream-generate-content]
+ [metabase.metabot.self.mistral :as mistral]
+ [metabase.metabot.self.moonshot :as moonshot]
+ [metabase.metabot.self.openai :as openai]
+ [metabase.metabot.self.openrouter :as openrouter]
+ [metabase.metabot.self.vllm :as vllm]
+ [metabase.metabot.self.xai :as xai]
+ [metabase.metabot.self.zai :as zai]
+ [metabase.test.fixtures :as fixtures]))
+
+(set! *warn-on-reflection* true)
+
+;; every-catalog-model-is-classified-test reads the provider registry, whose entries pass through
+;; llm.provider/provider-type -> hosted? -> premium-features/is-hosted?, a license lookup backed by the app db.
+;; metabase.metabot.self-test initializes :db for the same two calls.
+(use-fixtures :once (fixtures/initialize :db))
+
+(def ^:private omitted
+ "What a surface sends when it caps nothing: the field is absent from the body, rather than present and nil.
+ The difference is the whole point — an absent field lets the model run to its own limit."
+ ::omitted)
+
+(def ^:private stock-input
+ [{:role :user :content "hi"}])
+
+(defn- cap-for
+ "The output-token cap `surface`'s own request builder puts in the body for `model`, or [[omitted]].
+ `extra` is merged into the request opts."
+ ([surface model] (cap-for surface model nil))
+ ([surface model extra]
+ (let [opts (merge {:model model :input stock-input} extra)]
+ ;; Every branch calls a pure public builder — no `with-redefs`, so the deftests here stay `^:parallel`.
+ ;; The grouped branches are the surfaces that share a builder and differ only in id spelling. Dispatch from
+ ;; the `*-raw` fns down to these builders stays covered by the existing bedrock, azure and google tests.
+ (case surface
+ ;; Anthropic Messages: direct, the managed proxy (resolve-model-ref has already dropped `anthropic/`),
+ ;; Bedrock's `anthropic.` prefix, and Azure's bare admin-cased deployment name.
+ (:anthropic :managed :bedrock-anthropic :azure-anthropic)
+ (get (claude/claude-request-body opts) :max_tokens omitted)
+
+ ;; Vertex names the model in the URL and dates it with `@`, which raw-predict rewrites before delegating.
+ :vertex-claude (get (raw-predict/request-body model opts) :max_tokens omitted)
+ :deepseek (get (deepseek/deepseek-request-body opts) :max_tokens omitted)
+
+ ;; OpenAI Responses: direct, and Azure's deployment name.
+ (:openai :azure-openai)
+ (get (openai/openai-request-body opts) :max_output_tokens omitted)
+
+ ;; Bedrock's `openai.` prefix, through the mantle adaptation bedrock-raw applies to the same builder.
+ :bedrock-openai
+ (get (bedrock/->mantle-openai-body (openai/openai-request-body opts)) :max_output_tokens omitted)
+
+ :openrouter (get (openrouter/openrouter-request-body opts) :max_tokens omitted)
+ :zai (get (zai/zai-request-body opts) :max_tokens omitted)
+ :moonshot (get (moonshot/moonshot-request-body opts) :max_tokens omitted)
+ :mistral (get (mistral/mistral-request-body opts) :max_tokens omitted)
+ :vllm (get (vllm/vllm-request-body opts) :max_tokens omitted)
+ ;; xAI sends the cap as max_completion_tokens
+ :xai (get (xai/xai-request-body opts) :max_completion_tokens omitted)
+ :gemini (get-in (stream-generate-content/request-body opts)
+ [:generationConfig :maxOutputTokens] omitted)))))
+
+(def ^:private expected-chat-caps
+ "The cap each surface sends for each model it serves, keyed by surface and then by that surface's own id spelling.
+
+ Written out literally rather than derived from the default, so a change to the default, or to which surfaces send
+ none, fails here instead of drifting quietly. [[omitted]] means no cap is sent at all."
+ {;; Anthropic Messages, direct: the catalog's own dated and undated spellings
+ :anthropic {"claude-fable-5-1" 32000
+ "claude-fable-5" 32000
+ "claude-opus-5-5" 32000
+ "claude-opus-5" 32000
+ "claude-opus-4-8" 32000
+ "claude-opus-4-7" 32000
+ "claude-opus-4-6" 32000
+ "claude-sonnet-5-5" 32000
+ "claude-sonnet-5" 32000
+ "claude-sonnet-4-6" 32000
+ "claude-opus-4-5-20251101" 32000
+ "claude-sonnet-4-5-20250929" 32000
+ "claude-haiku-4-5-20251001" 32000
+ "claude-opus-4-1-20250805" 32000
+ ;; an id no catalog names, and an admin-named deployment: the Messages API needs a cap, so
+ ;; they get the same default rather than an omission
+ "claude-fable-6" 32000
+ "my-deployment-3" 32000}
+ ;; OpenAI Responses, direct: no default cap is sent (see openai/openai-request-body)
+ :openai {"gpt-6-astra" omitted
+ "gpt-5.6-sol" omitted
+ "gpt-5.6-terra" omitted
+ "gpt-5.6-luna" omitted
+ "gpt-5.5" omitted
+ "gpt-5.5-pro" omitted
+ "gpt-5.4" omitted
+ "gpt-5.4-pro" omitted
+ "gpt-5.4-mini" omitted
+ "gpt-4o" omitted}
+ ;; the managed proxy: resolve-model-ref hands the adapter the bare model
+ :managed {"claude-sonnet-4-6" 32000}
+ ;; Bedrock: vendor-prefixed
+ :bedrock-anthropic {"anthropic.claude-fable-5-1" 32000
+ "anthropic.claude-fable-5" 32000
+ "anthropic.claude-opus-5-5" 32000
+ "anthropic.claude-opus-5" 32000
+ "anthropic.claude-opus-4-8" 32000
+ "anthropic.claude-opus-4-7" 32000
+ "anthropic.claude-sonnet-5-5" 32000
+ "anthropic.claude-sonnet-5" 32000
+ "anthropic.claude-haiku-4-5" 32000}
+ :bedrock-openai {"openai.gpt-6-astra" 32000
+ "openai.gpt-5.4" 32000
+ "openai.gpt-5.4-2026-03-05" 32000
+ "openai.gpt-5.5" 32000
+ "openai.gpt-5.5-2026-04-23" 32000}
+ ;; Azure: the bare deployment name, dateless, and cased however the admin named it
+ :azure-anthropic {"claude-fable-5-1" 32000
+ "claude-fable-5" 32000
+ "claude-opus-5-5" 32000
+ "claude-opus-5" 32000
+ "claude-opus-4-8" 32000
+ "claude-opus-4-7" 32000
+ "claude-opus-4-6" 32000
+ "claude-sonnet-5-5" 32000
+ "claude-sonnet-5" 32000
+ "claude-sonnet-4-6" 32000
+ "claude-opus-4-5" 32000
+ "claude-sonnet-4-5" 32000
+ "claude-haiku-4-5" 32000
+ "claude-opus-4-1" 32000
+ "Claude-Opus-4-8" 32000
+ "Claude-Haiku-4-5" 32000}
+ :azure-openai {"gpt-6-astra" omitted
+ "gpt-5.6-sol" omitted
+ "gpt-5.6-terra" omitted
+ "gpt-5.6-luna" omitted
+ "gpt-5.6" omitted
+ "gpt-5.5-pro" omitted
+ "gpt-5.5" omitted
+ "gpt-5.4-pro" omitted
+ "gpt-5.4-mini" omitted
+ "gpt-5.4-nano" omitted
+ "gpt-5.4" omitted
+ "GPT-5.5" omitted}
+ ;; Vertex: partner ids, dated with `@`
+ :vertex-claude {"claude-fable-5-1" 32000
+ "claude-fable-5" 32000
+ "claude-opus-5-5" 32000
+ "claude-opus-5" 32000
+ "claude-opus-4-6" 32000
+ "claude-sonnet-5-5" 32000
+ "claude-sonnet-5" 32000
+ "claude-sonnet-4-6" 32000
+ "claude-haiku-4-5@20251001" 32000}
+ :gemini {"google/gemini-3.5-flash" 32000
+ "google/gemini-3.6-flash" 32000
+ "google/gemini-3.7-flash" 32000
+ "google/gemini-2.0-flash" 32000}
+ :deepseek {"deepseek-v4-pro" 32000
+ "deepseek-flash" 32000}
+ ;; OpenRouter: dotted Claude versions, dated DeepSeek snapshots
+ :openrouter {"anthropic/claude-fable-5.1" 32000
+ "anthropic/claude-fable-5" 32000
+ "anthropic/claude-opus-5" 32000
+ "anthropic/claude-opus-4.8" 32000
+ "anthropic/claude-opus-4.7" 32000
+ "anthropic/claude-opus-4.6" 32000
+ "anthropic/claude-sonnet-5" 32000
+ "anthropic/claude-sonnet-4.6" 32000
+ "anthropic/claude-opus-4.5" 32000
+ "anthropic/claude-sonnet-4.5" 32000
+ "anthropic/claude-haiku-4.5" 32000
+ "anthropic/claude-opus-4.1" 32000
+ "deepseek/deepseek-v4-pro" 32000
+ "deepseek/deepseek-v4-pro-0813" 32000
+ "deepseek/deepseek-v4-flash-0731" 32000
+ "z-ai/glm-5.3" 32000
+ "z-ai/glm-5.2" 32000
+ "qwen/qwen3.8-max" 32000
+ ;; the id qwen/qwen3.8-max resolves to on OpenRouter today
+ "qwen/qwen3.8-max-0902" 32000
+ ;; unlike on OpenAI direct and Azure, GPT models get the default here
+ "openai/gpt-5.6-sol" 32000
+ "openai/gpt-5.6-terra" 32000
+ "openai/gpt-5.6-luna" 32000
+ "openai/gpt-5.5" 32000
+ "openai/gpt-5.5-pro" 32000
+ "openai/gpt-5.4" 32000
+ "openai/gpt-5.4-mini" 32000
+ "openai/gpt-5.4-pro" 32000
+ "mistralai/mistral-medium-3-5" 32000
+ "moonshotai/kimi-k3" 32000
+ ;; ids no catalog names get the default too
+ "anthropic/claude-fable-6" 32000
+ "openai/gpt-4o" 32000}
+ :zai {"glm-5.3" 32000
+ "glm-5.2" 32000
+ "glm-4.7" 32000}
+ :moonshot {"kimi-k3" 32000
+ "kimi-k2.6" 32000
+ "kimi-k2.7-code" 32000}
+ :mistral {"mistral-medium-3-5" 32000
+ "mistral-medium-latest" 32000}
+ :xai {"grok-4.7" 32000
+ "grok-4.3" 32000}
+ ;; no default cap without a known context window, which the pure builder is not given
+ ;; (see vllm/vllm-request-body)
+ :vllm {"Qwen/Qwen3-32B" omitted}})
+
+(deftest ^:parallel chat-cap-is-the-flat-default-test
+ (testing "each surface sends the same default cap for every model, or deliberately none"
+ (doseq [[surface models] expected-chat-caps
+ [model expected] models]
+ (testing (str surface " " model)
+ (is (= expected (cap-for surface model)))))))
+
+(deftest ^:parallel caller-cap-always-wins-test
+ (testing "a task cap the caller names is sent verbatim, never replaced by the default"
+ (doseq [[surface models] expected-chat-caps
+ model (keys models)]
+ (testing (str surface " " model)
+ (is (= 512 (cap-for surface model {:max-tokens 512})))))))
+
+(deftest ^:parallel forced-tool-call-without-a-caller-cap-gets-the-chat-cap-test
+ (testing "the default cap clears the 2048 forced-tool-call floors, so the floor changes nothing"
+ (are [surface model expected] (= expected (cap-for surface model {:schema {:type "object"}}))
+ :openrouter "qwen/qwen3.8-max" 32000
+ :openrouter "anthropic/claude-fable-5" 32000
+ :openrouter "z-ai/glm-5.3" 32000
+ :zai "glm-5.3" 32000
+ :moonshot "kimi-k3" 32000
+ :gemini "google/gemini-3.7-flash" 32000))
+ (testing "vLLM with no known window still sends no cap: a floor raises a caller's cap, it never adds one"
+ (is (= omitted (cap-for :vllm "Qwen/Qwen3-32B" {:schema {:type "object"}})))))
+
+(defn- registry-models
+ "The default and mini model the provider registry names for `type-name`, nils dropped: Azure has no default
+ (its model is a deployment name), and the single-model types have no cheaper tier."
+ [type-name]
+ (keep identity [(llm.provider/default-model type-name) (llm.provider/mini-model type-name)]))
+
+(defn- catalog-pairs
+ "Every `[surface id]` the shipped catalogs offer, in the spelling that surface's builder receives.
+ Bedrock and Azure each serve two wire families, so their ids are split by the prefix that selects one."
+ []
+ (concat
+ (for [id (concat (keys claude/supported-models) (registry-models "anthropic"))] [:anthropic id])
+ (for [id (concat (keys openai/supported-models) (registry-models "openai"))] [:openai id])
+ (for [id (concat (keys deepseek/supported-models) (registry-models "deepseek"))] [:deepseek id])
+ (for [id (concat (keys moonshot/supported-models) (registry-models "moonshot"))] [:moonshot id])
+ (for [id (concat (keys mistral/supported-models) (registry-models "mistral"))] [:mistral id])
+ (for [id (concat (keys zai/supported-models) (registry-models "zai"))] [:zai id])
+ (for [id (concat (keys openrouter/supported-models) (registry-models "openrouter"))] [:openrouter id])
+ (for [id (concat (keys xai/supported-models) (registry-models "xai"))] [:xai id])
+ (for [id (concat (keys bedrock/supported-models) (registry-models "bedrock"))]
+ [(if (str/starts-with? id "anthropic.") :bedrock-anthropic :bedrock-openai) id])
+ ;; Azure lists no models; its context map is the closest thing to a catalog, and its keys are spelled the way
+ ;; deployment names reach the builder.
+ (for [id (keys @#'azure/model-context-windows)]
+ [(if (str/starts-with? id "claude-") :azure-anthropic :azure-openai) id])
+ (for [id (keys google.models/catalog)] [:gemini id])
+ ;; Google's and the managed provider's catalogs are fixed in the registry. Google's Gemini ids reach the
+ ;; builder whole; its Anthropic ids and the managed one lose the publisher segment on the way.
+ (for [id (concat (map :id (llm.provider/fixed-models "google")) (registry-models "google"))]
+ (if (str/starts-with? id "google/")
+ [:gemini id]
+ [:vertex-claude (llm.provider/model-ref->model id)]))
+ (for [id (concat (map :id (llm.provider/fixed-models "metabase")) (registry-models "metabase"))]
+ [:managed (llm.provider/model-ref->model id)])))
+
+(deftest ^:parallel every-catalog-model-is-classified-test
+ (testing "every model we ship has a deliberate expected cap, so a new catalog entry cannot slip in unclassified"
+ ;; One-directional on purpose: expected-chat-caps also holds unknown ids and alternate spellings that no
+ ;; catalog names.
+ (let [classified (set (for [[surface models] expected-chat-caps
+ model (keys models)]
+ [surface model]))]
+ (doseq [pair (catalog-pairs)]
+ (testing (pr-str pair)
+ (is (contains? classified pair)))))))
diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.clj
index 9c767af1706..d02a9ff6cc7 100644
--- a/test/metabase/metabot/self/claude_test.clj
+++ b/test/metabase/metabot/self/claude_test.clj
@@ -678,23 +678,23 @@
4096 {:schema schema :max-tokens 4096}
512 {:max-tokens 512}))))
-(deftest ^:parallel every-supported-model-has-a-ceiling-test
- (doseq [[id {:keys [display-name max-tokens]}] @#'claude/supported-models]
- (is (pos-int? max-tokens) id)
+(deftest ^:parallel every-supported-model-has-a-display-name-test
+ (doseq [[id {:keys [display-name]}] @#'claude/supported-models]
(is (seq display-name) id)))
(deftest claude-max-tokens-test
(mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key "sk-ant-test"]
(let [max-tokens #(:max_tokens (capture-claude-request-body!
(merge {:input [{:role :user :content "hi"}]} %)))]
- (are [opts tokens] (= tokens (max-tokens opts))
- {:model "claude-opus-4-8"} 128000
- {:model "claude-fable-5-1"} 128000
- {:model "claude-haiku-4-5-20251001"} 64000
- {:model "claude-opus-4-8" :max-tokens 32000} 32000
- ;; Bedrock ids reach us vendor-prefixed
- {:model "anthropic.claude-opus-4-8"} 128000
- {:model "my-deployment-3"} @#'claude/default-max-tokens))))
+ (testing "every model gets the same default cap, and a caller's own cap wins"
+ (are [opts tokens] (= tokens (max-tokens opts))
+ {:model "claude-opus-4-8"} 32000
+ {:model "claude-fable-5-1"} 32000
+ {:model "claude-haiku-4-5-20251001"} 32000
+ {:model "claude-opus-4-8" :max-tokens 4096} 4096
+ ;; Bedrock ids reach us vendor-prefixed
+ {:model "anthropic.claude-opus-4-8"} 32000
+ {:model "my-deployment-3"} 32000)))))
(deftest claude-auto-cache-breakpoint-test
(mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key "sk-ant-test"]
diff --git a/test/metabase/metabot/self/deepseek_test.clj b/test/metabase/metabot/self/deepseek_test.clj
index 0a52f06e6c8..4e51339a794 100644
--- a/test/metabase/metabot/self/deepseek_test.clj
+++ b/test/metabase/metabot/self/deepseek_test.clj
@@ -124,8 +124,8 @@
:tools [(metabot.tu/get-time-tool)]})]
(testing "the model defaults to deepseek-v4-pro"
(is (= "deepseek-v4-pro" (:model body))))
- (testing "an unknown model id falls back to the shared max_tokens ceiling"
- (is (= 64000 (:max_tokens body))))
+ (testing "max_tokens is the Claude request body's default cap"
+ (is (= 32000 (:max_tokens body))))
(is (true? (:stream body)))
(testing "the system prompt is sent as cache-marked content blocks"
(is (=? [{:type "text" :text "You are Metabot." :cache_control {:type "ephemeral"}}]
diff --git a/test/metabase/metabot/self/google/raw_predict_test.clj b/test/metabase/metabot/self/google/raw_predict_test.clj
index d30bd348c0a..684672528b6 100644
--- a/test/metabase/metabot/self/google/raw_predict_test.clj
+++ b/test/metabase/metabot/self/google/raw_predict_test.clj
@@ -41,27 +41,16 @@
:input [{:role :user :content "hi"}]})
:speed)))))
-(deftest ^:parallel request-body-dated-model-resolves-max-tokens-test
- (testing "a model dated in the platform's `@` spelling resolves to the same max_tokens ceiling as its
- direct-API `-` spelling, rather than falling back to the unknown-model default"
- (is (= 32000
- (:max_tokens (raw-predict/request-body "claude-opus-4-1@20250805"
- {:input [{:role :user :content "hi"}]}))))))
-
-(deftest ^:parallel request-body-bare-model-resolves-max-tokens-test
- (testing "an undated model ID is already in the direct-API spelling and resolves as-is"
- (is (= 128000
- (:max_tokens (raw-predict/request-body "claude-sonnet-4-6"
- {:input [{:role :user :content "hi"}]}))))))
-
-(deftest ^:parallel request-body-unknown-model-gets-default-max-tokens-test
- (testing "a model outside the Claude adapter's knowledge falls back to the safe default ceiling"
- (is (= 64000
- (:max_tokens (raw-predict/request-body "claude-not-a-real-model"
- {:input [{:role :user :content "hi"}]}))))))
+(deftest ^:parallel request-body-default-max-tokens-test
+ (testing "every model gets the Claude request body's default max_tokens, dated, bare, or unknown"
+ (are [model-id] (= 32000
+ (:max_tokens (raw-predict/request-body model-id {:input [{:role :user :content "hi"}]})))
+ "claude-opus-4-1@20250805"
+ "claude-sonnet-4-6"
+ "claude-not-a-real-model")))
(deftest ^:parallel request-body-explicit-max-tokens-wins-test
- (testing "an explicit :max-tokens overrides the model's ceiling"
+ (testing "an explicit :max-tokens overrides the default"
(is (= 4096
(:max_tokens (raw-predict/request-body "claude-opus-4-1@20250805"
{:input [{:role :user :content "hi"}]
diff --git a/test/metabase/metabot/self/google/stream_generate_content_test.clj b/test/metabase/metabot/self/google/stream_generate_content_test.clj
index 649e9f49930..32cd6cec10f 100644
--- a/test/metabase/metabot/self/google/stream_generate_content_test.clj
+++ b/test/metabase/metabot/self/google/stream_generate_content_test.clj
@@ -235,14 +235,16 @@
(deftest ^:parallel request-body-system-instruction-test
(testing "the system prompt becomes systemInstruction, outside :contents"
(is (= {:systemInstruction {:parts [{:text "You are a helpful assistant."}]}
- :contents [{:role "user" :parts [{:text "hi"}]}]}
+ :contents [{:role "user" :parts [{:text "hi"}]}]
+ :generationConfig {:maxOutputTokens 32000}}
(sgc/request-body
{:system "You are a helpful assistant."
:input [{:role :user :content "hi"}]})))))
(deftest ^:parallel request-body-no-system-message-test
(testing "no systemInstruction is sent when system is not provided"
- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}
+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
+ :generationConfig {:maxOutputTokens 32000}}
(sgc/request-body {:input [{:role :user :content "hi"}]})))))
(deftest ^:parallel request-body-generation-config-test
@@ -254,9 +256,9 @@
:temperature 0.2})))))
(deftest ^:parallel request-body-generation-config-only-when-set-test
- (testing "maxOutputTokens is omitted when the caller doesn't pass max-tokens, letting the model use its own limit"
+ (testing "maxOutputTokens falls back to the default cap when the caller doesn't pass max-tokens"
(is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
- :generationConfig {:temperature 0.2}}
+ :generationConfig {:maxOutputTokens 32000 :temperature 0.2}}
(sgc/request-body {:input [{:role :user :content "hi"}]
:temperature 0.2}))))
(testing "temperature is omitted when the caller doesn't pass one, letting the model use its own default"
@@ -264,13 +266,15 @@
:generationConfig {:maxOutputTokens 256}}
(sgc/request-body {:input [{:role :user :content "hi"}]
:max-tokens 256}))))
- (testing "no generationConfig is sent when neither max-tokens nor temperature is provided"
- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}
+ (testing "generationConfig carries only the default cap when neither max-tokens nor temperature is provided"
+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
+ :generationConfig {:maxOutputTokens 32000}}
(sgc/request-body {:input [{:role :user :content "hi"}]})))))
(deftest ^:parallel request-body-tool-choice-without-tools-test
(testing "a tool_choice with no tools to choose from sends no toolConfig, which Gemini rejects on its own"
- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}
+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
+ :generationConfig {:maxOutputTokens 32000}}
(sgc/request-body {:input [{:role :user :content "hi"}]
:tool_choice "required"})))))
@@ -319,24 +323,26 @@
(deftest ^:parallel request-body-structured-output-test
(testing "a :schema forces a structured_output function call via mode ANY + allowedFunctionNames"
(let [schema {:type "object" :properties {:sql {:type "string"}}}]
- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
- :tools [{:functionDeclarations [{:name "structured_output"
- :description "Output structured data"
- :parametersJsonSchema schema}]}]
- :toolConfig {:functionCallingConfig {:mode "ANY"
- :allowedFunctionNames ["structured_output"]}}}
+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
+ :generationConfig {:maxOutputTokens 32000}
+ :tools [{:functionDeclarations [{:name "structured_output"
+ :description "Output structured data"
+ :parametersJsonSchema schema}]}]
+ :toolConfig {:functionCallingConfig {:mode "ANY"
+ :allowedFunctionNames ["structured_output"]}}}
(sgc/request-body
{:input [{:role :user :content "hi"}]
:schema schema}))))))
(deftest ^:parallel request-body-structured-output-replaces-tools-test
(testing "a :schema replaces the caller's tools and tool_choice"
- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
- :tools [{:functionDeclarations [{:name "structured_output"
- :description "Output structured data"
- :parametersJsonSchema {:type "object"}}]}]
- :toolConfig {:functionCallingConfig {:mode "ANY"
- :allowedFunctionNames ["structured_output"]}}}
+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
+ :generationConfig {:maxOutputTokens 32000}
+ :tools [{:functionDeclarations [{:name "structured_output"
+ :description "Output structured data"
+ :parametersJsonSchema {:type "object"}}]}]
+ :toolConfig {:functionCallingConfig {:mode "ANY"
+ :allowedFunctionNames ["structured_output"]}}}
(sgc/request-body
{:input [{:role :user :content "hi"}]
:tools [{:tool-name "get_weather" :doc "Get the weather."
@@ -361,15 +367,18 @@
:reasoning? false
:schema {:type "object"}}))))
(testing ":reasoning? false sends no thinkingConfig, leaving the server default"
- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}
+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
+ :generationConfig {:maxOutputTokens 32000}}
(sgc/request-body {:model "google/gemini-3.6-flash"
:input [{:role :user :content "hi"}]
:reasoning? false}))))
(testing "an off-catalog or absent model gets no thinkingConfig at all"
- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}
+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
+ :generationConfig {:maxOutputTokens 32000}}
(sgc/request-body {:model "google/gemini-2.5-flash"
:input [{:role :user :content "hi"}]})))
- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}
+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
+ :generationConfig {:maxOutputTokens 32000}}
(sgc/request-body {:input [{:role :user :content "hi"}]})))))
(deftest ^:parallel request-body-forced-tool-call-token-floor-test
@@ -385,11 +394,16 @@
:input [{:role :user :content "hi"}]
:schema {:type "object"}
:max-tokens 8000}))))
- (testing "an uncapped structured call stays uncapped — the floor raises, it never introduces a cap"
- (is (nil? (get-in (sgc/request-body {:model "google/gemini-3.7-flash"
- :input [{:role :user :content "hi"}]
- :schema {:type "object"}})
- [:generationConfig :maxOutputTokens]))))
+ (testing "an uncapped structured call takes the default cap, already above the floor"
+ (is (= 32000 (get-in (sgc/request-body {:model "google/gemini-3.7-flash"
+ :input [{:role :user :content "hi"}]
+ :schema {:type "object"}})
+ [:generationConfig :maxOutputTokens]))))
+ (testing "an off-catalog model's uncapped structured call takes the same default cap"
+ (is (= 32000 (get-in (sgc/request-body {:model "google/gemini-2.5-flash"
+ :input [{:role :user :content "hi"}]
+ :schema {:type "object"}})
+ [:generationConfig :maxOutputTokens]))))
(testing "an unforced call keeps the caller's cap"
(is (=? {:generationConfig {:maxOutputTokens 512}}
(sgc/request-body {:model "google/gemini-3.7-flash"
@@ -416,6 +430,24 @@
(is (=? {:generationConfig {:maxOutputTokens 512}}
(body-for "auto"))))))
+(deftest ^:parallel request-body-default-max-tokens-test
+ (let [input [{:role :user :content "hi"}]]
+ (testing "an uncapped call gets the default cap, on and off the catalog"
+ (are [model] (= 32000 (get-in (sgc/request-body {:model model :input input})
+ [:generationConfig :maxOutputTokens]))
+ "google/gemini-3.5-flash"
+ "google/gemini-3.6-flash"
+ "google/gemini-3.7-flash"
+ "google/gemini-2.5-flash"))
+ (testing "a request naming no model at all gets it too"
+ (is (= 32000 (get-in (sgc/request-body {:input input})
+ [:generationConfig :maxOutputTokens]))))
+ (testing "the caller's own task cap wins over the default"
+ (is (= 512 (get-in (sgc/request-body {:model "google/gemini-3.7-flash"
+ :input input
+ :max-tokens 512})
+ [:generationConfig :maxOutputTokens]))))))
+
;;; ──────────────────────────────────────────────────────────────────
;;; Streaming event conversion tests.
;;; ──────────────────────────────────────────────────────────────────
diff --git a/test/metabase/metabot/self/google_test.clj b/test/metabase/metabot/self/google_test.clj
index ebaf1dede8b..3e7d88cbd4f 100644
--- a/test/metabase/metabot/self/google_test.clj
+++ b/test/metabase/metabot/self/google_test.clj
@@ -161,7 +161,8 @@
:headers {"Content-Type" "application/json"}}
req))
(is (= {:contents [{:role "user" :parts [{:text "hi"}]}]
- :generationConfig {:thinkingConfig {:includeThoughts true}}}
+ :generationConfig {:maxOutputTokens 32000
+ :thinkingConfig {:includeThoughts true}}}
(json/decode+kw (:body req)))))))))
(deftest google-raw-regional-location-host-test
@@ -401,8 +402,7 @@
"the URL names the model; a model in the body is rejected by the platform"))))))
(deftest google-raw-anthropic-max-tokens-test
- (testing "the Anthropic model whitelist supplies max_tokens for a bare current-generation ID, and the @-versioned
- spelling of a dated ID resolves to the same entry the direct Anthropic adapter uses"
+ (testing "an Anthropic model gets the default max_tokens, whether its ID is bare or @-versioned"
(mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
llm.settings/llm-google-service-account-key nil
llm.settings/llm-google-project-id "my-project"
@@ -411,11 +411,11 @@
self.core/reducible-with-api-errors (fn [r _ _] r)
debug/capture-stream (fn [r _] r)
http/request (fn [req] {:body req})]
- (doseq [[model max-tokens] {"anthropic/claude-sonnet-4-6" 128000
- "anthropic/claude-fable-5" 128000
- "anthropic/claude-haiku-4-5@20251001" 64000}]
+ (doseq [model ["anthropic/claude-sonnet-4-6"
+ "anthropic/claude-fable-5"
+ "anthropic/claude-haiku-4-5@20251001"]]
(testing model
- (is (= max-tokens
+ (is (= 32000
(:max_tokens (json/decode+kw (:body (google-raw {:model model
:input [{:role :user :content "hi"}]}))))))))))))
@@ -1322,12 +1322,14 @@
:as :stream}
post-req))
(testing "the body is a Chat Completions request with an empty model, since the endpoint serves one"
- (is (=? {:model ""
- :stream true
- :stream_options {:include_usage true}
- :messages [{:role "user" :content "hi"}]
- :max_tokens pos-int?}
- (json/decode+kw (:body post-req))))))))
+ (let [body (json/decode+kw (:body post-req))]
+ (is (=? {:model ""
+ :stream true
+ :stream_options {:include_usage true}
+ :messages [{:role "user" :content "hi"}]}
+ body))
+ (testing "and, like vLLM chat with an unknown context window, no output cap"
+ (is (not (contains? body :max_tokens)))))))))
(deftest google-raw-dedicated-endpoint-host-test
(testing "a dedicated endpoint is served on the DNS name its resource reports, in either spelling Google uses, also when the base URL names Google's host for the location"
diff --git a/test/metabase/metabot/self/moonshot_test.clj b/test/metabase/metabot/self/moonshot_test.clj
index 47fda381a50..3a2623246f5 100644
--- a/test/metabase/metabot/self/moonshot_test.clj
+++ b/test/metabase/metabot/self/moonshot_test.clj
@@ -123,7 +123,8 @@
2048 {:model "kimi-k3" :input [{:role :user :content "hi"}] :max-tokens 512
:tools [(metabot.tu/get-time-tool)] :tool_choice "required"}
4096 {:model "kimi-k3" :input [{:role :user :content "hi"}] :schema schema :max-tokens 4096}
- nil {:model "kimi-k3" :input [{:role :user :content "hi"}] :schema schema}
+ ;; uncapped by the caller, so the default cap applies — already above the floor
+ 32000 {:model "kimi-k3" :input [{:role :user :content "hi"}] :schema schema}
512 {:model "kimi-k3" :input [{:role :user :content "hi"}] :max-tokens 512}
512 {:model "kimi-k2.6" :input [{:role :user :content "hi"}] :schema schema :max-tokens 512}
512 {:model "kimi-k2.6" :input [{:role :user :content "hi"}] :max-tokens 512
diff --git a/test/metabase/metabot/self/openrouter_test.clj b/test/metabase/metabot/self/openrouter_test.clj
index bc7329df5d8..f6bc8521a8e 100644
--- a/test/metabase/metabot/self/openrouter_test.clj
+++ b/test/metabase/metabot/self/openrouter_test.clj
@@ -244,10 +244,11 @@
;; safety net — its reasoning cannot be disabled and bills against the same budget as the
;; tool call; theory vs practice in openrouter/forced-tool-call-token-floor
(are [expected opts] (= expected (:max_tokens (body-for (assoc opts :model "qwen/qwen3.8-max-0902"))))
- 2048 {:schema {:type "object"} :max-tokens 512}
- 4096 {:schema {:type "object"} :max-tokens 4096}
- nil {:schema {:type "object"}}
- 512 {:max-tokens 512}))
+ 2048 {:schema {:type "object"} :max-tokens 512}
+ 4096 {:schema {:type "object"} :max-tokens 4096}
+ ;; uncapped by the caller, so the default cap applies — already far above the floor
+ 32000 {:schema {:type "object"}}
+ 512 {:max-tokens 512}))
(testing "a non-mandatory model keeps its cap — it got the disable instead"
(let [body (body-for {:model "anthropic/claude-sonnet-4.6" :schema {:type "object"} :max-tokens 512})]
(is (= 512 (:max_tokens body)))
diff --git a/test/metabase/metabot/self/vllm_test.clj b/test/metabase/metabot/self/vllm_test.clj
index d8a2f6e37b2..ce0f6a752da 100644
--- a/test/metabase/metabot/self/vllm_test.clj
+++ b/test/metabase/metabot/self/vllm_test.clj
@@ -37,14 +37,56 @@
;;; vllm-request-body
;;; ──────────────────────────────────────────────────────────────────
-(deftest ^:parallel request-body-applies-default-max-tokens-test
- (testing "an explicit max_tokens is always sent — without one vLLM falls back to the whole remaining context window"
- (is (= (llm.settings/llm-max-tokens)
- (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
- :input [{:role :user :content "hi"}]}))))))
+(deftest ^:parallel request-body-sends-no-default-max-tokens-test
+ (testing "no max_tokens is sent without a caller cap or a known window — vLLM would subtract one from the
+ admissible prompt, and nothing says how much room is left"
+ (is (not (contains? (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]})
+ :max_tokens)))))
+
+(defn- cap-for-window
+ "The `max_tokens` [[vllm/vllm-request-body]] sends for `opts` on a `window`-token context window, or
+ `::omitted`."
+ [opts window]
+ (get (vllm/vllm-request-body (merge {:model "vllm-test" :input [{:role :user :content "hi"}]} opts) window)
+ :max_tokens
+ ::omitted))
+
+(defn- long-input
+ "A one-message input of `n` copies of `s`."
+ [s n]
+ [{:role :user :content (apply str (repeat n s))}])
+
+(deftest ^:parallel request-body-sends-the-chat-cap-when-it-fits-the-window-test
+ (testing "with a known window, the agent loop gets the flat chat cap"
+ (is (= self.core/chat-max-output-tokens (cap-for-window {} 131072)))))
+
+(deftest ^:parallel request-body-drops-a-cap-that-does-not-fit-the-window-test
+ (testing "the chat cap alone is larger than the smallest window preflight accepts, so it is not sent"
+ (is (= ::omitted (cap-for-window {} 16384))))
+ (testing "the prompt counts: a cap that fits beside a short prompt is dropped beside a long one"
+ (is (= 32000 (cap-for-window {} 65536)))
+ (is (= ::omitted (cap-for-window {:input (long-input "x" 70000)} 65536))))
+ (testing "a caller's cap is dropped the same way rather than lowered"
+ (is (= ::omitted (cap-for-window {:input (long-input "x" 70000) :max-tokens 512} 32768))))
+ (testing "digits count one token each, as Qwen3 tokenizes them"
+ ;; Qwen3 gives each digit its own token, so 40000 digits are 40000 tokens and 32000 does not fit in
+ ;; 65536; at 2 bytes per token the estimate was about 21000 and sent a cap that vLLM rejects
+ (is (= ::omitted (cap-for-window {:input (long-input "7" 40000)} 65536))))
+ (testing "a non-ASCII prompt is counted in bytes, not characters"
+ ;; 25000 CJK characters are 75000 UTF-8 bytes, about 76000 estimated tokens, so 32000 no longer fits;
+ ;; counted as characters they would leave room for it
+ (is (= ::omitted (cap-for-window {:input (long-input "表" 25000)} 65536)))))
+
+(deftest ^:parallel request-body-reasoning-floor-on-the-smallest-window-test
+ (testing "the 16384 reasoning floor fills the whole 16384 window preflight accepts, so a raised cap is dropped
+ rather than sent into a 400 — even on a 512-token title call"
+ (is (= ::omitted (cap-for-window {:max-tokens 512 :credentials reasoning-credentials} 16384))))
+ (testing "and on a window with room for it, the floor still applies"
+ (is (= 16384 (cap-for-window {:max-tokens 512 :credentials reasoning-credentials} 32768)))))
(deftest ^:parallel request-body-caller-max-tokens-wins-test
- (testing "a caller-supplied max-tokens is not overridden by the default"
+ (testing "a caller-supplied max-tokens is sent as-is"
(is (= 128
(:max_tokens (vllm/vllm-request-body {:model "vllm-test"
:input [{:role :user :content "hi"}]
@@ -64,7 +106,13 @@
:input [{:role :user :content "hi"}]
:tools [(metabot.tu/get-time-tool)]
:tool_choice "required"
- :max-tokens 128})))))))
+ :max-tokens 128}))))))
+ (testing "a forced tool call with no caller cap and no known window sends none — a floor raises a cap,
+ it never adds one"
+ (is (not (contains? (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :schema {:type "object"}})
+ :max_tokens)))))
(deftest ^:parallel request-body-floor-never-lowers-a-ceiling-test
(testing "a caller-supplied ceiling above the floor is left alone"
@@ -84,17 +132,23 @@
:max-tokens 128}))))))
(deftest ^:parallel request-body-raises-max-tokens-for-a-reasoning-model-test
- (testing "the agent path forces nothing and supplies no ceiling, so a reasoning model would otherwise get
- the shared 4096 default for thinking, answer, and tool call combined"
+ (testing "a reasoning model's caller cap is raised, since thinking, answer, and tool call share one budget"
(is (= 16384
(:max_tokens (vllm/vllm-request-body {:model "vllm-test"
:input [{:role :user :content "hi"}]
+ :max-tokens 128
:credentials reasoning-credentials}))))
- (testing "and a model the probe found does not reason keeps the default"
- (is (= (llm.settings/llm-max-tokens)
+ (testing "and a model the probe found does not reason keeps the caller's cap"
+ (is (= 128
(:max_tokens (vllm/vllm-request-body {:model "vllm-test"
:input [{:role :user :content "hi"}]
- :credentials credentials})))))))
+ :max-tokens 128
+ :credentials credentials}))))))
+ (testing "with no known window the agent path supplies no cap, so a reasoning model is sent none either"
+ (is (not (contains? (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials reasoning-credentials})
+ :max_tokens)))))
(deftest ^:parallel request-body-reasoning-floor-outranks-the-forced-tool-call-floor-test
(testing "a reasoning model has to clear its thinking before the forced call, so the higher floor wins"
@@ -572,6 +626,103 @@
:input [{:role :user :content "hi"}]
:credentials credentials}))))))
+;;; ──────────────────────────────────────────────────────────────────
+;;; Context-window lookup
+;;; ──────────────────────────────────────────────────────────────────
+
+(defn- fresh-credentials
+ "Credentials for a server no other test has looked up, so the context-window cache starts empty for it."
+ []
+ {:base-url (str "http://vllm-" (random-uuid) ".internal:8000/v1")})
+
+(defn- chat-with-catalog!
+ "Run `vllm-raw` `n` times against a stub whose `/models` answers with `(models-response req)`. Return the
+ `/models` requests it saw and the last chat body it was sent."
+ [credentials models-response & {:keys [n] :or {n 1}}]
+ (let [lookups (atom [])
+ chat-body (atom nil)]
+ (with-redefs [self.core/sse-reducible (fn [_] (reify clojure.lang.IReduceInit
+ (reduce [_ _rf init] init)))
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [{:keys [url body] :as req}]
+ (if (re-find #"/models$" (str url))
+ (do (swap! lookups conj req)
+ (models-response req))
+ (do (reset! chat-body (json/decode+kw body))
+ {:body nil})))]
+ (dotimes [_ n]
+ (vllm/vllm-raw {:model "vllm-test" :input [{:role :user :content "hi"}] :credentials credentials})))
+ {:lookups @lookups :chat-body @chat-body}))
+
+(defn- catalog
+ "A `/models` stub answering with `entries`."
+ [& entries]
+ (constantly {:status 200 :body {:data (vec entries)}}))
+
+(deftest vllm-raw-sizes-max-tokens-from-the-served-window-test
+ (testing "the window is read from the request's own model entry, not the first one listed"
+ (let [{:keys [lookups chat-body]} (chat-with-catalog! (fresh-credentials)
+ (catalog {:id "other" :max_model_len 16384}
+ {:id "vllm-test" :max_model_len 131072}))]
+ (is (= 32000 (:max_tokens chat-body)))
+ (testing "with a short timeout, since the chat request waits for the lookup"
+ (is (=? [{:method :get :socket-timeout 5000 :connection-timeout 5000}] lookups)))))
+ (testing "a served window too small for the cap sends none"
+ (is (not (contains? (:chat-body (chat-with-catalog! (fresh-credentials)
+ (catalog {:id "vllm-test" :max_model_len 16384})))
+ :max_tokens)))))
+
+(deftest vllm-raw-treats-an-unreported-window-as-unknown-test
+ (testing "no cap without a window: the model is not listed, or its entry has no `max_model_len`"
+ (doseq [models-response [(catalog {:id "other" :max_model_len 131072})
+ (catalog {:id "vllm-test"})
+ (catalog {:id "vllm-test" :max_model_len "131072"})]]
+ (is (not (contains? (:chat-body (chat-with-catalog! (fresh-credentials) models-response)) :max_tokens)))))
+ (testing "a failed lookup is not an error: the chat request is still sent, with no cap"
+ (doseq [failure [(SocketTimeoutException. "Read timed out")
+ (ConnectException. "Connection refused")
+ (ex-info "clj-http: status 404" {:status 404 :body "Not Found"})]]
+ (let [{:keys [chat-body]} (chat-with-catalog! (fresh-credentials) (fn [_] (throw failure)))]
+ (is (= "vllm-test" (:model chat-body)))
+ (is (not (contains? chat-body :max_tokens)))))))
+
+(deftest vllm-raw-caches-the-served-window-test
+ (testing "one lookup serves the following requests to the same server and model"
+ (let [{:keys [lookups chat-body]} (chat-with-catalog! (fresh-credentials)
+ (catalog {:id "vllm-test" :max_model_len 131072})
+ :n 3)]
+ (is (= 1 (count lookups)))
+ (is (= 32000 (:max_tokens chat-body)))))
+ (testing "a failure is cached too, so a server that cannot answer does not slow every request"
+ (is (= 1 (count (:lookups (chat-with-catalog! (fresh-credentials)
+ (fn [_] (throw (SocketTimeoutException. "Read timed out")))
+ :n 3)))))))
+
+(defn- lookups-at!
+ "The number of `/models` lookups one `vllm-raw` call makes at clock time `t`."
+ [clock t credentials models-response]
+ (reset! clock t)
+ (count (:lookups (chat-with-catalog! credentials models-response))))
+
+(deftest vllm-raw-keeps-an-answered-lookup-longer-than-a-failed-one-test
+ (let [clock (atom 0)
+ answered-ttl @#'vllm/answered-lookup-ttl-ms
+ failed-ttl @#'vllm/failed-lookup-ttl-ms]
+ (mt/with-dynamic-fn-redefs [vllm/now-ms (fn [] @clock)]
+ (testing "an answer is reused until the long TTL runs out, with or without a window"
+ (doseq [response [(catalog {:id "vllm-test" :max_model_len 131072})
+ (catalog {:id "vllm-test"})]]
+ (let [creds (fresh-credentials)]
+ (is (= [1 0 1] [(lookups-at! clock 0 creds response)
+ (lookups-at! clock (inc failed-ttl) creds response)
+ (lookups-at! clock answered-ttl creds response)])))))
+ (testing "a failed lookup is tried again after the short TTL"
+ (let [creds (fresh-credentials)
+ response (fn [_] (throw (SocketTimeoutException. "Read timed out")))]
+ (is (= [1 0 1] [(lookups-at! clock 0 creds response)
+ (lookups-at! clock (dec failed-ttl) creds response)
+ (lookups-at! clock failed-ttl creds response)])))))))
+
;;; ──────────────────────────────────────────────────────────────────
;;; list-models
;;; ──────────────────────────────────────────────────────────────────
diff --git a/test/metabase/metabot/self/xai_test.clj b/test/metabase/metabot/self/xai_test.clj
index aa6abd0365f..9bcd7de9dad 100644
--- a/test/metabase/metabot/self/xai_test.clj
+++ b/test/metabase/metabot/self/xai_test.clj
@@ -36,6 +36,8 @@
(let [body (xai/xai-request-body {:input input :max-tokens 512})]
(is (= 512 (:max_completion_tokens body)))
(is (not (contains? body :max_tokens)))))
+ (testing "a caller with no cap gets the chat cap, not xAI's 128,000 default"
+ (is (= 32000 (:max_completion_tokens (xai/xai-request-body {:input input})))))
(testing "chat keeps each model's default reasoning effort"
(is (not (contains? (xai/xai-request-body {:input input :tools [(metabot.tu/get-time-tool)]})
:reasoning_effort))))
diff --git a/test/metabase/metabot/self/zai_test.clj b/test/metabase/metabot/self/zai_test.clj
index ec420686e6a..832e9d82abc 100644
--- a/test/metabase/metabot/self/zai_test.clj
+++ b/test/metabase/metabot/self/zai_test.clj
@@ -105,8 +105,7 @@
(testing "thinking-only models reject the disable (error 1210): no directive, reasoning_effort instead"
(let [body (zai/zai-request-body {:model "glm-5.3" :input input})]
(is (not (contains? body :thinking)))
- (is (= "max" (:reasoning_effort body)))
- (is (not (contains? body :max_tokens)))))
+ (is (= "max" (:reasoning_effort body)))))
(testing "a schema drops the effort to low and raises a title-sized cap to the floor"
(is (=? {:reasoning_effort "low"
:max_tokens 2048}
@@ -140,6 +139,22 @@
(is (not (contains? body :reasoning_effort)))
(is (= 512 (:max_tokens body)))))))
+(deftest ^:parallel request-body-default-max-tokens-test
+ (let [input [{:role :user :content "hi"}]]
+ (testing "an uncapped call gets the default cap, whatever the model"
+ (are [model] (= 32000 (:max_tokens (zai/zai-request-body {:model model :input input})))
+ "glm-5.3"
+ "glm-5.2"
+ "glm-4.7"))
+ (testing "the caller's own task cap wins over the default"
+ (is (= 512 (:max_tokens (zai/zai-request-body {:model "glm-5.3"
+ :input input
+ :max-tokens 512})))))
+ (testing "a forced call takes the default, which is already above the floor"
+ (is (= 32000 (:max_tokens (zai/zai-request-body {:model "glm-5.3"
+ :input input
+ :schema {:type "object"}})))))))
+
(deftest ^:parallel reasoning-model?-test
(are [model expected] (= expected (zai/reasoning-model? model))
"glm-5.2" true
diff --git a/test/metabase/settings/models/setting_test.clj b/test/metabase/settings/models/setting_test.clj
index 4653754c0ad..519e78a4217 100644
--- a/test/metabase/settings/models/setting_test.clj
+++ b/test/metabase/settings/models/setting_test.clj
@@ -1371,6 +1371,10 @@
(is (= "Setting name 'retired-setting' is retired; use a different name instead"
(ex-message e))))))))
+(deftest ^:parallel llm-max-tokens-is-retired-test
+ (testing "the removed llm-max-tokens setting's name cannot be reused by a new setting"
+ (is (contains? @#'setting/retired-setting-names "llm-max-tokens"))))
+
(deftest duplicated-setting-name
(testing "can re-register a setting in the same ns (redefining or reloading ns)"
(is (defsetting foo (deferred-tru "A testing setting") :visibility :public :encryption :when-encryption-key-set))