The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Send one flat max_tokens for Metabot chat, and retire the llm-max-tokens setting (#82436)
Paolo Rascunàbbf81b462c3d
diff --git a/docs/configuring-metabase/config-template.md b/docs/configuring-metabase/config-template.mdindex 51c24fc997c..91b679050a7 100644--- a/docs/configuring-metabase/config-template.md+++ b/docs/configuring-metabase/config-template.md@@ -227,7 +227,6 @@ config:llm-google-oauth-access-token: nullllm-google-project-id: nullllm-google-service-account-key: null- llm-max-tokens: 4096llm-metabot-provider: anthropic/claude-sonnet-4-6llm-mini-model: nullllm-mistral-api-base-url: https://api.mistral.ai/v1diff --git a/docs/configuring-metabase/environment-variables.md b/docs/configuring-metabase/environment-variables.mdindex c25b7eb20fb..c6ef18ce363 100644--- a/docs/configuring-metabase/environment-variables.md+++ b/docs/configuring-metabase/environment-variables.md@@ -1523,14 +1523,6 @@ A Google Cloud service account key JSON for the Gemini Enterprise Agent PlatformBacked by the google connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.-### `MB_LLM_MAX_TOKENS`--- Type: integer-- Default: `4096`-- [Configuration file name](./config-file.md): `llm-max-tokens`--Maximum tokens for LLM responses.-### `MB_LLM_METABOT_PROVIDER`- Type: stringdiff --git a/src/metabase/llm/settings.clj b/src/metabase/llm/settings.cljindex b785ecd2e77..2bbbd470817 100644--- a/src/metabase/llm/settings.clj+++ b/src/metabase/llm/settings.clj@@ -446,13 +446,6 @@:default true:export? true)-(defsetting llm-max-tokens- (deferred-tru "Maximum tokens for LLM responses.")- :type :integer- :default 4096- :visibility :settings-manager- :export? false)-(defsetting llm-request-timeout-ms(deferred-tru(str "Socket (inter-byte read) timeout in milliseconds for LLM API requests. "diff --git a/src/metabase/metabot/self/bedrock.clj b/src/metabase/metabot/self/bedrock.cljindex 99841f9b469..f3c1b57f965 100644--- a/src/metabase/metabot/self/bedrock.clj+++ b/src/metabase/metabot/self/bedrock.clj@@ -418,6 +418,16 @@[body](dissoc body :cache_control))+(defn ->mantle-openai-body+ "Adapt a canonical OpenAI Responses request body for the mantle endpoint.++ A body without `max_output_tokens` gets [[core/chat-max-output-tokens]]."+ [body]+ ;; Unlike OpenAI direct, an unset cap costs quota here: mantle's quota check counts "the value of `max_tokens` (or+ ;; the model-specific maximum if `max_tokens` is not set)"+ ;; (https://docs.aws.amazon.com/bedrock/latest/userguide/quotas-mantle.html).+ (update body :max_output_tokens #(or % core/chat-max-output-tokens)))+(mu/defn bedrock-raw"Perform a streaming request to Bedrock: the mantle endpoint for a mantle catalog ID, `bedrock-runtime` for aClaude model only it serves (see [[runtime-model?]]).@@ -439,7 +449,7 @@(claude/claude-request-body (assoc opts :model (runtime-base-model model)))):read-stream runtime-events}:openai {:path "/openai/v1/responses"- :req (openai/openai-request-body opts)})]+ :req (->mantle-openai-body (openai/openai-request-body opts))})](adapter/stream! (if (= :runtime family) runtime-provider provider) opts{:path path:body reqdiff --git a/src/metabase/metabot/self/claude.clj b/src/metabase/metabot/self/claude.cljindex 15f232483e8..ea2d4101d51 100644--- a/src/metabase/metabot/self/claude.clj+++ b/src/metabase/metabot/self/claude.clj@@ -347,24 +347,20 @@(def supported-models"Anthropic chat models offered in the Metabot model picker, keyed by model id.`list-models` returns the intersection of this map with the account's `/v1/models` catalog."- {"claude-fable-5-1" {:display-name "Claude Fable 5.1" :max-tokens 128000 :context-window 1000000}- "claude-fable-5" {:display-name "Claude Fable 5" :max-tokens 128000 :context-window 1000000}- "claude-opus-5-5" {:display-name "Claude Opus 5.5" :max-tokens 128000 :context-window 1000000}- "claude-opus-5" {:display-name "Claude Opus 5" :max-tokens 128000 :context-window 1000000}- "claude-opus-4-8" {:display-name "Claude Opus 4.8" :max-tokens 128000 :context-window 1000000}- "claude-opus-4-7" {:display-name "Claude Opus 4.7" :max-tokens 128000 :context-window 1000000}- "claude-opus-4-6" {:display-name "Claude Opus 4.6" :max-tokens 128000 :context-window 1000000}- "claude-opus-4-5-20251101" {:display-name "Claude Opus 4.5" :max-tokens 64000 :context-window 200000}- "claude-opus-4-1-20250805" {:display-name "Claude Opus 4.1" :max-tokens 32000 :context-window 200000}- "claude-sonnet-5-5" {:display-name "Claude Sonnet 5.5" :max-tokens 128000 :context-window 1000000}- "claude-sonnet-5" {:display-name "Claude Sonnet 5" :max-tokens 128000 :context-window 1000000}- "claude-sonnet-4-6" {:display-name "Claude Sonnet 4.6" :max-tokens 128000 :context-window 1000000}- "claude-sonnet-4-5-20250929" {:display-name "Claude Sonnet 4.5" :max-tokens 64000 :context-window 200000}- "claude-haiku-4-5-20251001" {:display-name "Claude Haiku 4.5" :max-tokens 64000 :context-window 200000}})--(def ^:private default-max-tokens- "`max_tokens` for an unresolved model — low enough to be safe on any of them."- 64000)+ {"claude-fable-5-1" {:display-name "Claude Fable 5.1" :context-window 1000000}+ "claude-fable-5" {:display-name "Claude Fable 5" :context-window 1000000}+ "claude-opus-5-5" {:display-name "Claude Opus 5.5" :context-window 1000000}+ "claude-opus-5" {:display-name "Claude Opus 5" :context-window 1000000}+ "claude-opus-4-8" {:display-name "Claude Opus 4.8" :context-window 1000000}+ "claude-opus-4-7" {:display-name "Claude Opus 4.7" :context-window 1000000}+ "claude-opus-4-6" {:display-name "Claude Opus 4.6" :context-window 1000000}+ "claude-opus-4-5-20251101" {:display-name "Claude Opus 4.5" :context-window 200000}+ "claude-opus-4-1-20250805" {:display-name "Claude Opus 4.1" :context-window 200000}+ "claude-sonnet-5-5" {:display-name "Claude Sonnet 5.5" :context-window 1000000}+ "claude-sonnet-5" {:display-name "Claude Sonnet 5" :context-window 1000000}+ "claude-sonnet-4-6" {:display-name "Claude Sonnet 4.6" :context-window 1000000}+ "claude-sonnet-4-5-20250929" {:display-name "Claude Sonnet 4.5" :context-window 200000}+ "claude-haiku-4-5-20251001" {:display-name "Claude Haiku 4.5" :context-window 200000}})(mu/defn list-models :- adapter/ModelListing"List the Anthropic chat models supported by this adapter, by intersecting [[supported-models]] with the@@ -384,11 +380,6 @@[model](str/replace-first (u/lower-case-en (str model)) #"^anthropic\." ""))-(defn- model-max-tokens- "The `max_tokens` ceiling for `model`, or nil when it isn't one we know."- [model]- (get-in supported-models [(strip-vendor-prefix model) :max-tokens]))-(mu/defn context-window-tokens :- [:maybe :int]"The input context window for `model`, or nil when it isn't one we know."[model :- [:maybe :string]]@@ -496,7 +487,8 @@(add-tools-cache-breakpoint all-tools)all-tools)](cond-> {:model model- :max_tokens (or max-tokens (model-max-tokens model) default-max-tokens)+ ;; required by the Messages API (https://platform.claude.com/docs/en/api/messages), so never omitted+ :max_tokens (or max-tokens core/chat-max-output-tokens):stream true:cache_control {:type "ephemeral"}:messages messages}diff --git a/src/metabase/metabot/self/core.clj b/src/metabase/metabot/self/core.cljindex 0065a527983..f55347e8b99 100644--- a/src/metabase/metabot/self/core.clj+++ b/src/metabase/metabot/self/core.clj@@ -274,6 +274,26 @@[:fast? {:optional true} [:maybe :boolean]][:prompt-cache-key {:optional true} [:maybe :string]]])+(def chat-max-output-tokens+ "The output-token cap for Metabot chat, sent when the caller passes no `:max-tokens` — only the agent loop does.++ Sized for Metabot's chat rather than for any model: production chat output has a p99.9 of about 7,300 tokens, so+ 32000 truncates only a runaway generation, and it is at or below every catalog model's documented maximum, the+ lowest being Claude Opus 4.1's 32,000+ (https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-4-1.html).++ A constant rather than a setting: the output distribution is the same on every instance, telemetry records no+ finish reason so a lowered value would truncate tool calls invisibly, and some surfaces deliberately send no cap+ (see the OpenAI builder, and Google Model Garden endpoints, which reach the vLLM builder with no context window)+ or send it only when it fits the context window (see the vLLM builder), so one global knob would mislead.++ Where a provider counts the cap against the context window, the cap also takes room from the prompt. Mistral+ (https://docs.mistral.ai/api/endpoint/chat) and Moonshot (https://platform.kimi.ai/docs/api/chat) reject a+ request whose prompt plus cap exceeds the window, and both send this cap on every request: on a 262,144-token+ window a prompt over 230,144 tokens fails while the context meter shows about 88%. The agent loop resends the+ full history and does not compact it, so a long enough conversation reaches this limit."+ 32000)+(defn mkid"Generate a random id"[]diff --git a/src/metabase/metabot/self/google/raw_predict.clj b/src/metabase/metabot/self/google/raw_predict.cljindex b5853bfa60e..6da7cf08149 100644--- a/src/metabase/metabot/self/google/raw_predict.clj+++ b/src/metabase/metabot/self/google/raw_predict.clj@@ -23,8 +23,8 @@(defn- direct-api-model-id"The direct-API spelling of a platform model ID.The Gemini Agent Platform versions a dated model as `{model}@{date}` where the Messages API spells it- `{model}-{date}`. The Claude adapter's model knowledge (max_tokens ceilings, thinking support) is keyed by the- direct spelling."+ `{model}-{date}`. The Claude adapter's model knowledge (thinking support, context windows) is keyed by the direct+ spelling."[model-id](str/replace (str model-id) "@" "-"))diff --git a/src/metabase/metabot/self/google/stream_generate_content.clj b/src/metabase/metabot/self/google/stream_generate_content.cljindex 335416d4049..a2f78a0a0a3 100644--- a/src/metabase/metabot/self/google/stream_generate_content.clj+++ b/src/metabase/metabot/self/google/stream_generate_content.clj@@ -158,7 +158,9 @@2048)(mu/defn request-body- "Builds the `streamGenerateContent` request body for an LLM request."+ "Builds the `streamGenerateContent` request body for an LLM request.++ A request that carries no `:max-tokens` is capped at [[core/chat-max-output-tokens]]."[{:keys [system input tools schema tool_choice temperature max-tokens model reasoning?]:or {reasoning? true}} :- core/LLMRequestOpts](let [fdecls (when (seq tools) (mapv tool->function-declaration tools))@@ -181,13 +183,14 @@;; The chat path streams to the browser: ask for the thought summaries the;; chain-of-thought UI renders, and leave the default thinking level alone.reasoning? {:includeThoughts true}))+ max-tokens (or max-tokens core/chat-max-output-tokens);; Safety net: the forced tool call must survive the un-disableable thinking spend, which- ;; Gemini bills against maxOutputTokens (see [[forced-tool-call-token-floor]]). Only an- ;; existing cap is raised, and only where a tool call is actually forced — the chat path- ;; sends no cap at all. Independent of :reasoning?, because a catalog model thinks whether- ;; or not we asked it to.+ ;; Gemini bills against maxOutputTokens (see [[forced-tool-call-token-floor]]). The cap is+ ;; raised only where a tool call is actually forced. The default cap is well above the+ ;; floor, so this bites only on a smaller caller cap. Independent of :reasoning?, because a+ ;; catalog model thinks whether or not we asked it to.max-tokens (cond-> max-tokens- (and max-tokens forced? (models/reasoning-model? model))+ (and forced? (models/reasoning-model? model))(max forced-tool-call-token-floor))gen-config (cond-> {}max-tokens (assoc :maxOutputTokens max-tokens)diff --git a/src/metabase/metabot/self/mistral.clj b/src/metabase/metabot/self/mistral.cljindex e5436d7ebdd..90c25426a72 100644--- a/src/metabase/metabot/self/mistral.clj+++ b/src/metabase/metabot/self/mistral.clj@@ -101,8 +101,10 @@- Prompt caching is opt-in per request via `prompt_cache_key` (cache reads bill at 10% of the input price), so a`:prompt-cache-key` — the conversation id — is forwarded when present.- Whitelisted models get a `reasoning_effort` directive and replay their in-turn reasoning as think chunks- (see [[think-message]])."- [{:keys [model prompt-cache-key reasoning? schema] :as opts+ (see [[think-message]]).++ A caller that names no `:max-tokens` gets [[core/chat-max-output-tokens]]."+ [{:keys [model prompt-cache-key reasoning? schema max-tokens] :as opts:or {model default-model reasoning? true}} :- core/LLMRequestOpts];; mistral-medium-3-5 accepts exactly "high" and "none" — the server 400s the other four;; enum values, enumerating these two (probed 2026-09-01) — and its server default sends NO@@ -117,7 +119,7 @@;; from the replayed input, honoring the LLMRequestOpts contract.thinking? (and whitelisted? reasoning? (not schema))](-> (chat-completions/request-body- (assoc opts :model model)+ (assoc opts :model model :max-tokens (or max-tokens core/chat-max-output-tokens))(when thinking? {:reasoning-part->message think-message}))(dissoc :stream_options)(cond-> prompt-cache-key (assoc :prompt_cache_key prompt-cache-key)diff --git a/src/metabase/metabot/self/moonshot.clj b/src/metabase/metabot/self/moonshot.cljindex c9177e50b0b..904f5c53f71 100644--- a/src/metabase/metabot/self/moonshot.clj+++ b/src/metabase/metabot/self/moonshot.clj@@ -91,9 +91,9 @@2048)(defn- floor-max-tokens- "Raises an existing `max_tokens` cap to [[forced-tool-call-token-floor]]; an uncapped body stays uncapped."+ "Raises the `max_tokens` cap to [[forced-tool-call-token-floor]]."[body]- (cond-> body (:max_tokens body) (update :max_tokens max forced-tool-call-token-floor)))+ (update body :max_tokens max forced-tool-call-token-floor))(defn- reasoning-message"The replayed assistant message for a coalesced in-turn :reasoning part.@@ -125,8 +125,10 @@otherwise; their in-turn reasoning is replayed (see [[reasoning-message]]) and forced tool calls get a`max_tokens` floor (see [[forced-tool-call-token-floor]]).- **`prompt_cache_key`.** Moonshot caching is automatic and hits without it, but a `:prompt-cache-key` — the- conversation id — is forwarded when present."- [{:keys [model prompt-cache-key reasoning? schema tool_choice] :as opts+ conversation id — is forwarded when present.++ A caller that names no `:max-tokens` gets [[core/chat-max-output-tokens]]."+ [{:keys [model prompt-cache-key reasoning? schema tool_choice max-tokens] :as opts:or {model default-model reasoning? true}} :- core/LLMRequestOpts];; kimi-k3 always thinks — there is no off switch, so `reasoning_effort` ("low" | "high" | "max", server default;; "max") is the only knob (https://platform.kimi.ai/docs/guide/kimi-k3-quickstart). "max" on the chat path pins@@ -148,7 +150,7 @@thinking? (and whitelisted? reasoning?(if thinking-only? (not schema) (not forced?)))](-> (chat-completions/request-body- (assoc opts :model model)+ (assoc opts :model model :max-tokens (or max-tokens core/chat-max-output-tokens));; Replay only where the dialect mandates it: k3's Preserved Thinking. k2.6 cannot use;; `thinking.keep "all"` — that mode obliges the caller to send back EVERY historical assistant;; message's reasoning_content, but reasoning parts are never persisted across turns, so we coulddiff --git a/src/metabase/metabot/self/openai.clj b/src/metabase/metabot/self/openai.cljindex b2564e309c3..50cb66756a0 100644--- a/src/metabase/metabot/self/openai.clj+++ b/src/metabase/metabot/self/openai.clj@@ -316,7 +316,9 @@(reasoning-model? model))(mu/defn openai-request-body- "Build the OpenAI Responses API request body for an LLM request."+ "Build the OpenAI Responses API request body for an LLM request.++ `max_output_tokens` is sent only when the caller passes `:max-tokens`."[{:keys [model system input tools schema tool_choice temperature max-tokens reasoning?]:or {model default-model reasoning? true}} :- core/LLMRequestOpts](let [input (cond->> input@@ -338,6 +340,11 @@tool_choice tool_choice:else "auto"):tools all-tools)+ ;; No default cap: OpenAI and Azure meter a sent cap against the rate limit — "Your rate limit is calculated+ ;; as the maximum of max_tokens and the estimated number of tokens"+ ;; (https://developers.openai.com/api/docs/guides/rate-limits,+ ;; https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/quota). Both pages name max_tokens; nothing+ ;; says the Responses field max_output_tokens is metered differently.max-tokens (assoc :max_output_tokens max-tokens);; encrypted_content lets us replay reasoning items across tool-calldiff --git a/src/metabase/metabot/self/openrouter.clj b/src/metabase/metabot/self/openrouter.cljindex 2504ade2d72..83728f9d7f9 100644--- a/src/metabase/metabot/self/openrouter.clj+++ b/src/metabase/metabot/self/openrouter.clj@@ -244,10 +244,10 @@[body {:keys [model reasoning? schema] :or {reasoning? true}}](let [forced? (or (some? schema) (= "required" (:tool_choice body)));; Safety net: the mandatory tool call must survive the un-disableable thinking spend- ;; (theory vs practice in [[forced-tool-call-token-floor]]); only an existing cap is- ;; raised, and only where a tool call is actually forced.+ ;; (theory vs practice in [[forced-tool-call-token-floor]]); the cap is raised only+ ;; where a tool call is actually forced.body (cond-> body- (and (reasoning-mandatory? model) forced? (:max_tokens body))+ (and (reasoning-mandatory? model) forced?)(update :max_tokens max forced-tool-call-token-floor))](if-not (= :renderable (reasoning-class model))body@@ -274,10 +274,17 @@`:temperature` is dropped for models that reject it (see [[model-supports-temperature?]]). Gating it in the sharedbuilder instead would apply these OpenRouter-specific rules to every Chat Completions adapter, including vLLM,- whose model names are customer-chosen free text."- [{:keys [model system] :as opts+ whose model names are customer-chosen free text.++ A caller that names no `:max-tokens` gets [[core/chat-max-output-tokens]]."+ [{:keys [model system max-tokens] :as opts:or {model default-model}} :- core/LLMRequestOpts]- (-> (cond-> (chat-completions/request-body (assoc opts :model model))+ ;; `openai/*` models get the default too: the rate-limit metering that keeps+ ;; [[metabase.metabot.self.openai/openai-request-body]] from sending one is OpenAI's and Azure's, and uncapped,+ ;; OpenRouter substitutes a per-endpoint default (probed 2026-09-17 on deepseek/deepseek-v4-pro: 16384 on one+ ;; endpoint, 32768 on another).+ (-> (cond-> (chat-completions/request-body+ (assoc opts :model model :max-tokens (or max-tokens core/chat-max-output-tokens)))(and system (anthropic-model? model))(update-in [:messages 0 :content] claude/system->cached-content-blocks)diff --git a/src/metabase/metabot/self/vllm.clj b/src/metabase/metabot/self/vllm.cljindex 8130fc9a5ac..7663dd38482 100644--- a/src/metabase/metabot/self/vllm.clj+++ b/src/metabase/metabot/self/vllm.clj@@ -15,10 +15,12 @@[metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json]+ [metabase.util.log :as log][metabase.util.malli :as mu])(:import(java.io IOException)(java.net SocketTimeoutException)+ (java.nio.charset StandardCharsets)(java.util.concurrent ExecutionException)))(set! *warn-on-reflection* true)@@ -175,17 +177,44 @@2048)(def ^:private forced-tool-call-token-floor- "Smallest `max_tokens` a forced tool call is given, regardless of what the caller asked for — below- it a reasoning model spends the budget thinking and emits no tool call. Equal to- [[probe-max-tokens]], which [[preflight!]] already proves the served model can clear."+ "Smallest `max_tokens` a cap on a forced tool call is raised to — below it a reasoning model spends+ the budget thinking and emits no tool call. Equal to [[probe-max-tokens]], which [[preflight!]]+ already proves the served model can clear."probe-max-tokens)(def ^:private reasoning-model-token-floor- "Smallest `max_tokens` any request gets once [[preflight!]] has observed the served model reasoning.- Chat Completions bills thinking, answer, and tool call against one budget. Mirrors- `claude-request-body`."+ "Smallest `max_tokens` a cap is raised to once [[preflight!]] has observed the served model+ reasoning. Chat Completions bills thinking, answer, and tool call against one budget."16384)+(def ^:private prompt-overhead-tokens+ "Tokens [[estimated-prompt-tokens]] adds for what the chat template wraps around the messages and+ tools: role markers, a tool-use preamble, the generation prompt."+ 1024)++(def ^:private answered-lookup-ttl-ms+ "How long [[served-max-model-len]] reuses the answer of a `/v1/models` lookup that succeeded, with or+ without a window for the model.++ Five minutes, because a vLLM server is seldom restarted after its initial setup. The cost: after a+ restart with a smaller `--max-model-len`, a cap sized for the old window can get a 400 from vLLM for up+ to five minutes after the lookup that cached it. A server that lists no window (Ollama, LM Studio) does+ not start to list one, so it is asked again only at this interval too."+ (* 5 60 1000))++(def ^:private failed-lookup-ttl-ms+ "How long [[served-max-model-len]] reuses the nil window of a `/v1/models` lookup that failed: a+ timeout, a refused connection, or an HTTP error status.++ Shorter than [[answered-lookup-ttl-ms]], so a server that comes up is used soon. A `/v1/models` that is+ down still costs at most one [[window-lookup-timeout-ms]] lookup in this period."+ 30000)++(def ^:private window-lookup-timeout-ms+ "Connection and socket timeout of the `/v1/models` lookup behind [[served-max-model-len]]. The+ catalog is cheap to serve, and the chat request waits for this lookup."+ 5000)+(def ^:private default-temperature"Sampling temperature for a caller that supplies none. vLLM's own default is 1.0, which is wrong forthe tool-calling and SQL-generation work the agent loop does. The hosted providers pick a sane@@ -395,22 +424,139 @@;;; --------------------------------------------------- Requests -------------------------------------------------+(defonce ^:private window-cache+ ;; [base-url model] -> {:window n-or-nil, :expires-at epoch-ms}. Each entry carries its own expiry, because a+ ;; answered and a failed lookup live for different times (see [[lookup-entry]]). The keys are the servers+ ;; and models the connections name, so the map stays small without eviction.+ (atom {}))++(defn- now-ms+ "The current time in epoch milliseconds. A function, so a test can set the clock."+ []+ (System/currentTimeMillis))++(defn- window-entry+ "A [[window-cache]] entry for `window` that expires `ttl-ms` from now."+ [window ttl-ms]+ {:window window+ :expires-at (+ (now-ms) ttl-ms)})++(defn- live-entry+ "The [[window-cache]] entry for `k`, or nil when there is none or it has expired."+ [k]+ (let [entry (get @window-cache k)]+ (when (and entry (< (now-ms) (:expires-at entry)))+ entry)))++(defn- fetch-max-model-len+ "Fetch the context window the server at `credentials` advertises for `model`, or nil.++ vLLM's `/v1/models` entries carry `max_model_len`+ (https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/serve/engine/protocol.py, `ModelCard`).+ Nil when the catalog lists no entry with the model's id or no window for it (Ollama, LM Studio, TGI).+ Throws when the request fails."+ [credentials model]+ (let [res (adapter/request! provider+ {:credentials credentials :method :get :path "/models" :as :json}+ {:socket-timeout window-lookup-timeout-ms+ :connection-timeout window-lookup-timeout-ms})+ entry (u/seek #(= model (:id %)) (get-in res [:body :data]))]+ (when (pos-int? (:max_model_len entry))+ (:max_model_len entry))))++(defn- lookup-entry+ "Look up the window of `model` on the server at `credentials`, as a [[window-cache]] entry.++ An answer, with or without a window, lives for [[answered-lookup-ttl-ms]]. A failed lookup gives a nil+ window that lives for [[failed-lookup-ttl-ms]]: an unknown window only means no default cap, and the+ chat request that follows reports a real failure in its own words."+ [credentials model]+ (try+ (window-entry (fetch-max-model-len credentials model) answered-lookup-ttl-ms)+ (catch Exception e+ (log/debugf e "Could not read the context window of %s from the vLLM server" model)+ (window-entry nil failed-lookup-ttl-ms))))++(defn- served-max-model-len+ "The context window the vLLM server serves the request's model with, or nil when it is unknown.++ Read from the server rather than stored on the connection, so a restart with a different+ `--max-model-len` is followed within [[answered-lookup-ttl-ms]]. Lookups are cached per base URL and+ model, nil windows and failures too, so a server costs one lookup per [[answered-lookup-ttl-ms]], or per+ [[failed-lookup-ttl-ms]] while its lookup fails, not one per request. Two requests that miss at the same+ time both look up; the later answer wins."+ [{:keys [model credentials ai-proxy?]}]+ (let [base-url (:base-url credentials)+ k [base-url model]]+ (when-not (or ai-proxy? (str/blank? base-url))+ (:window (or (live-entry k)+ (let [entry (lookup-entry credentials model)]+ (swap! window-cache assoc k entry)+ entry))))))++(defn- estimated-prompt-tokens+ "A high estimate of the prompt tokens in the Chat Completions `body`.++ The UTF-8 bytes of its JSON `messages` and `tools`, plus [[prompt-overhead-tokens]]. One byte counts as+ one token: the server can host any model, and no token of a byte-level BPE or byte-fallback tokenizer+ is shorter than one byte. That is an upper bound, not a typical rate. Metabot prose measures 4 to 5+ bytes per token, but rows of numbers measure 1.0 on Qwen3, which gives each digit its own token, and+ below 2 on a tokenizer that groups up to three digits. A high estimate is the safe side:+ [[output-cap]] then sends no cap, where a low one would send a cap that does not fit."+ [body]+ (+ prompt-overhead-tokens+ (alength (.getBytes ^String (json/encode (select-keys body [:messages :tools]))+ StandardCharsets/UTF_8))))++(defn- raise-to-floors+ "Raise `cap` to [[forced-tool-call-token-floor]] on a `forced?` call, and to+ [[reasoning-model-token-floor]] on a reasoning connection."+ [cap forced? credentials]+ (cond-> cap+ forced? (max forced-tool-call-token-floor)+ (reasoning-connection? credentials) (max reasoning-model-token-floor)))++(defn- output-cap+ "The `max_tokens` to send with `body`, or nil to send none.++ For example, on a 131072-token window the agent loop's 48 KB first prompt gets 32000; on a 32768-token+ window it gets none, and vLLM stops at the remaining window, about 21000 tokens.++ The cap is the caller's `max-tokens`, or [[core/chat-max-output-tokens]] when the `window` is known,+ raised by [[raise-to-floors]]. It is sent only when [[estimated-prompt-tokens]] plus the cap fits the+ window: vLLM subtracts a sent cap from the admissible prompt and returns a 400 when the prompt no longer+ fits (https://github.com/vllm-project/vllm/blob/main/vllm/renderers/params.py). A cap that does not fit+ is dropped, not lowered. Uncapped, vLLM generates up to the remaining window, which then bounds the+ output by the exact prompt length rather than by an estimate.++ With an unknown window, only a caller's cap is sent."+ [body max-tokens forced? credentials window]+ (let [cap (some-> (or max-tokens (when window core/chat-max-output-tokens))+ (raise-to-floors forced? credentials))]+ (when (and cap+ (or (nil? window)+ (<= (+ (estimated-prompt-tokens body) cap) window)))+ cap)))+(mu/defn vllm-request-body"Build the Chat Completions request body for an LLM request.- Matches what [[chat-completions/request-body]] emits, except that `max_tokens` is always sent —- without a ceiling vLLM falls back to the remaining context window, so one looping small model- consumes the whole budget in a single call — and is raised to- [[forced-tool-call-token-floor]] or [[reasoning-model-token-floor]] where either applies, and- `temperature` falls back to [[default-temperature]]. All three stay adapter-local rather than- moving into the shared builder, which would also change Z.AI, Mistral, and OpenRouter."- [{:keys [max-tokens temperature schema tool_choice credentials] :as opts} :- core/LLMRequestOpts]- (let [forced? (or (some? schema) (= "required" (some-> tool_choice name)))]- (assoc (chat-completions/request-body (cond-> opts- (nil? temperature) (assoc :temperature default-temperature)))- :max_tokens (cond-> (or max-tokens (llm/llm-max-tokens))- forced? (max forced-tool-call-token-floor)- (reasoning-connection? credentials) (max reasoning-model-token-floor)))))+ Matches what [[chat-completions/request-body]] emits, except that `max_tokens` is the one+ [[output-cap]] picks for the context window `max-model-len` (nil when unknown), and `temperature`+ falls back to [[default-temperature]]. Both stay adapter-local rather than moving into the shared+ builder, which would also change Z.AI, Mistral, and OpenRouter.++ Pure: [[vllm-raw]] looks the window up. The 1-arity, which Model Garden endpoints use, has none."+ ([opts :- core/LLMRequestOpts]+ (vllm-request-body opts nil))+ ([{:keys [max-tokens temperature schema tool_choice credentials] :as opts} :- core/LLMRequestOpts+ max-model-len :- [:maybe pos-int?]]+ (let [forced? (or (some? schema) (= "required" (some-> tool_choice name)))+ body (chat-completions/request-body (cond-> (dissoc opts :max-tokens)+ (nil? temperature) (assoc :temperature default-temperature)))+ cap (output-cap body max-tokens forced? credentials max-model-len)]+ (cond-> body+ cap (assoc :max_tokens cap)))))(defn- stream-io-ex"The vLLM error for a transport failure while *consuming* a response stream. Tagged@@ -467,14 +613,15 @@"Perform a streaming request to a vLLM server's Chat Completions API.Opts map takes `:credentials` (`{:base-url ... :api-key ...}`) from the connection serving thisrequest, and throws without a base URL.- `:ai-proxy?` is not supported for vLLM and throws when true."+ `:ai-proxy?` is not supported for vLLM and throws when true.+ Reads the served context window first, through [[served-max-model-len]]'s cache, to size `max_tokens`."[{:keys [model credentials] :as opts} :- core/LLMRequestOpts](when (str/blank? model)(throw (missing-model-ex)))(let [timeout-ms (llm/llm-vllm-request-timeout-ms)](adapter/stream! provider opts{:path "/chat/completions"- :body (vllm-request-body opts)+ :body (vllm-request-body opts (served-max-model-len opts)):request-options (inference-timeouts):wrap-stream #(io-guarded % timeout-ms);; clj-http raises an `IOException` only when there is no response at all, so thediff --git a/src/metabase/metabot/self/xai.clj b/src/metabase/metabot/self/xai.cljindex c6233324f8d..7a20c554592 100644--- a/src/metabase/metabot/self/xai.clj+++ b/src/metabase/metabot/self/xai.clj@@ -61,17 +61,20 @@xAI's dialect matches what [[chat-completions/request-body]] emits, except:- The output cap is sent as `max_completion_tokens`, since xAI deprecates `max_tokens`. It caps visible output- only, not reasoning or tool calls, so a small cap cannot cut off a forced tool call.+ only, not reasoning or tool calls, so a small cap cannot cut off a forced tool call. A caller that names no+ `:max-tokens` gets [[core/chat-max-output-tokens]], since xAI's own default is 128,000 visible tokens+ (https://docs.x.ai/developers/rest-api-reference/inference/chat-completions).- The structured path, and a caller opting out of reasoning, get the model's `:lowest-effort` from[[supported-models]], which turns reasoning off on Grok 4.3 and lowers it to `low` on Grok 4.7. Chat keeps eachmodel's default. Models off the allow-list get no effort at all, since some of them (Grok 4.20) reject theparameter.- A `:prompt-cache-key`, the conversation id, is forwarded as `prompt_cache_key`, which routes a conversation'srequests to the same server so its prompt cache hits."- [{:keys [model prompt-cache-key reasoning? schema] :as opts+ [{:keys [model max-tokens prompt-cache-key reasoning? schema] :as opts:or {model default-model reasoning? true}} :- core/LLMRequestOpts](let [lowest-effort (get-in supported-models [model :lowest-effort])]- (-> (chat-completions/request-body (assoc opts :model model))+ (-> (chat-completions/request-body+ (assoc opts :model model :max-tokens (or max-tokens core/chat-max-output-tokens)))(set/rename-keys {:max_tokens :max_completion_tokens})(cond-> (and lowest-effort (or (some? schema) (not reasoning?)))(assoc :reasoning_effort lowest-effort)diff --git a/src/metabase/metabot/self/zai.clj b/src/metabase/metabot/self/zai.cljindex 887a69b705d..b866d910ca1 100644--- a/src/metabase/metabot/self/zai.clj+++ b/src/metabase/metabot/self/zai.clj@@ -91,9 +91,10 @@adding Z.AI's `thinking` directive: enabled only where a whitelisted model's reasoning renders, disabledotherwise. A [[thinking-only-model?]] rejects the directive and gets `reasoning_effort` instead — \"max\"where reasoning renders, \"low\" otherwise — plus a `max_tokens` floor on forced tool calls (see- [[forced-tool-call-token-floor]]). Z.AI documents only `tool_choice \"auto\"`, but `\"required\"` — which the- structured-output path relies on — is accepted and honored in practice, with thinking on."- [{:keys [model reasoning? schema tool_choice] :as opts+ [[forced-tool-call-token-floor]]). A caller that passes no cap of its own gets [[core/chat-max-output-tokens]]. Z.AI+ documents only `tool_choice \"auto\"`, but `\"required\"` — which the structured-output path relies on — is+ accepted and honored in practice, with thinking on."+ [{:keys [model reasoning? schema tool_choice max-tokens] :as opts:or {model default-model reasoning? true}} :- core/LLMRequestOpts];; Thinking is on by default server-side, at reasoning_effort "max";; (https://docs.z.ai/api-reference/llm/chat-completion), so "enabled" only makes the default@@ -109,12 +110,13 @@(let [thinking-only? (thinking-only-model? model)forced? (or (some? schema) (= "required" (some-> tool_choice name)))thinking? (and (reasoning-model? model) reasoning? (not schema))- body (chat-completions/request-body (assoc opts :model model))]+ max-tokens (or max-tokens core/chat-max-output-tokens)+ body (chat-completions/request-body (assoc opts :model model :max-tokens max-tokens))](cond-> bodythinking-only?(assoc :reasoning_effort (if thinking? "max" "low"))- (and thinking-only? forced? (:max_tokens body))+ (and thinking-only? forced?)(update :max_tokens max forced-tool-call-token-floor)(not thinking-only?)diff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.cljindex 9ad210d73d9..30c18eaa451 100644--- a/src/metabase/settings/models/setting.clj+++ b/src/metabase/settings/models/setting.clj@@ -77,7 +77,8 @@"metabot-enabled""ldap-sync-admin-group""user-recent-views"- "most-recently-viewed-dashboard"})+ "most-recently-viewed-dashboard"+ "llm-max-tokens"})#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}(def ^:dynamic *allow-retired-setting-names*diff --git a/test/metabase/llm/settings_test.clj b/test/metabase/llm/settings_test.cljindex 9115f9a1689..1995e65aaac 100644--- a/test/metabase/llm/settings_test.clj+++ b/test/metabase/llm/settings_test.clj@@ -397,14 +397,6 @@;;; ------------------------------------------- Settings Defaults Tests --------------------------------------------(deftest llm-max-tokens-test- (testing "default value is 4096"- (mt/with-temporary-setting-values [llm-max-tokens nil]- (is (= 4096 (llm.settings/llm-max-tokens)))))- (testing "can be overridden"- (mt/with-temporary-setting-values [llm-max-tokens 8192]- (is (= 8192 (llm.settings/llm-max-tokens))))))-(deftest llm-request-timeout-ms-test(testing "default value is 120000 (120 seconds)"(mt/with-temporary-setting-values [llm-request-timeout-ms nil]diff --git a/test/metabase/metabot/self/azure_test.clj b/test/metabase/metabot/self/azure_test.cljindex 1ff0c21862f..789e59398c7 100644--- a/test/metabase/metabot/self/azure_test.clj+++ b/test/metabase/metabot/self/azure_test.clj@@ -135,8 +135,8 @@:system [{:type "text" :text "be brief" :cache_control {:type "ephemeral"}}]:messages [{:role "user" :content [{:type "text" :text "hi"}]}]}body)))- (testing "a deployment name matches no model, so max_tokens falls back rather than being omitted"- (is (= 64000 (:max_tokens body))))))+ (testing "a deployment gets the default max_tokens rather than none"+ (is (= 32000 (:max_tokens body))))))(deftest openai-family-dispatches-to-responses-api-test(let [req (captured-raw-request! {:model "openai/gpt-5-deployment"diff --git a/test/metabase/metabot/self/bedrock_test.clj b/test/metabase/metabot/self/bedrock_test.cljindex 4caae8912f5..858b240dc4a 100644--- a/test/metabase/metabot/self/bedrock_test.clj+++ b/test/metabase/metabot/self/bedrock_test.clj@@ -322,12 +322,12 @@(json/decode+kw (:body (captured-raw-request! (merge {:input [{:role :user :content "hi"}]} opts)))))(deftest anthropic-model-max-tokens-test- (testing "the `anthropic.` prefix is stripped so the model's own ceiling resolves"+ (testing "anthropic.* models get the default cap, and the caller's own cap wins"(are [opts tokens] (= tokens (:max_tokens (captured-body! opts)))- {:model "anthropic.claude-opus-4-8"} 128000+ {:model "anthropic.claude-opus-4-8"} 32000{:model "anthropic.claude-opus-4-8" :max-tokens 128} 128))- (testing "openai.* models omit the field entirely"- (is (not (contains? (captured-body! {:model "openai.gpt-5.5"}) :max_output_tokens)))))+ (testing "openai.* models get the default cap too, unlike on OpenAI direct"+ (is (= 32000 (:max_output_tokens (captured-body! {:model "openai.gpt-5.5"}))))))(deftest reasoning-request-config-test(testing "anthropic models request adaptive summarized thinking, and only that"@@ -416,6 +416,14 @@:cache_control {:type "ephemeral"}:system [{:type "text" :text "s" :cache_control {:type "ephemeral"}}]})))))+(deftest ^:parallel mantle-openai-body-test+ (testing "adds the default max_output_tokens where the body has none"+ (is (= {:model "openai.gpt-5.5" :max_output_tokens 32000}+ (bedrock/->mantle-openai-body {:model "openai.gpt-5.5"}))))+ (testing "keeps a caller's own cap"+ (is (= {:model "openai.gpt-5.5" :max_output_tokens 512}+ (bedrock/->mantle-openai-body {:model "openai.gpt-5.5" :max_output_tokens 512})))))+;;; ──────────────────────────────────────────────────────────────────;;; Stream translation (xf selection by model family);;; ──────────────────────────────────────────────────────────────────diff --git a/test/metabase/metabot/self/chat_max_tokens_test.clj b/test/metabase/metabot/self/chat_max_tokens_test.cljnew file mode 100644index 00000000000..61262a5cba5--- /dev/null+++ b/test/metabase/metabot/self/chat_max_tokens_test.clj@@ -0,0 +1,294 @@+(ns metabase.metabot.self.chat-max-tokens-test+ "The output-token cap every surface sends when the caller names none.++ One namespace across every adapter, because the property is cross-provider: every surface sends the same default+ cap, whatever the model and however its provider spells the id, except the few that deliberately send none."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase.llm.provider :as llm.provider]+ [metabase.metabot.self.azure :as azure]+ [metabase.metabot.self.bedrock :as bedrock]+ [metabase.metabot.self.claude :as claude]+ [metabase.metabot.self.deepseek :as deepseek]+ [metabase.metabot.self.google.models :as google.models]+ [metabase.metabot.self.google.raw-predict :as raw-predict]+ [metabase.metabot.self.google.stream-generate-content :as stream-generate-content]+ [metabase.metabot.self.mistral :as mistral]+ [metabase.metabot.self.moonshot :as moonshot]+ [metabase.metabot.self.openai :as openai]+ [metabase.metabot.self.openrouter :as openrouter]+ [metabase.metabot.self.vllm :as vllm]+ [metabase.metabot.self.xai :as xai]+ [metabase.metabot.self.zai :as zai]+ [metabase.test.fixtures :as fixtures]))++(set! *warn-on-reflection* true)++;; every-catalog-model-is-classified-test reads the provider registry, whose entries pass through+;; llm.provider/provider-type -> hosted? -> premium-features/is-hosted?, a license lookup backed by the app db.+;; metabase.metabot.self-test initializes :db for the same two calls.+(use-fixtures :once (fixtures/initialize :db))++(def ^:private omitted+ "What a surface sends when it caps nothing: the field is absent from the body, rather than present and nil.+ The difference is the whole point — an absent field lets the model run to its own limit."+ ::omitted)++(def ^:private stock-input+ [{:role :user :content "hi"}])++(defn- cap-for+ "The output-token cap `surface`'s own request builder puts in the body for `model`, or [[omitted]].+ `extra` is merged into the request opts."+ ([surface model] (cap-for surface model nil))+ ([surface model extra]+ (let [opts (merge {:model model :input stock-input} extra)]+ ;; Every branch calls a pure public builder — no `with-redefs`, so the deftests here stay `^:parallel`.+ ;; The grouped branches are the surfaces that share a builder and differ only in id spelling. Dispatch from+ ;; the `*-raw` fns down to these builders stays covered by the existing bedrock, azure and google tests.+ (case surface+ ;; Anthropic Messages: direct, the managed proxy (resolve-model-ref has already dropped `anthropic/`),+ ;; Bedrock's `anthropic.` prefix, and Azure's bare admin-cased deployment name.+ (:anthropic :managed :bedrock-anthropic :azure-anthropic)+ (get (claude/claude-request-body opts) :max_tokens omitted)++ ;; Vertex names the model in the URL and dates it with `@`, which raw-predict rewrites before delegating.+ :vertex-claude (get (raw-predict/request-body model opts) :max_tokens omitted)+ :deepseek (get (deepseek/deepseek-request-body opts) :max_tokens omitted)++ ;; OpenAI Responses: direct, and Azure's deployment name.+ (:openai :azure-openai)+ (get (openai/openai-request-body opts) :max_output_tokens omitted)++ ;; Bedrock's `openai.` prefix, through the mantle adaptation bedrock-raw applies to the same builder.+ :bedrock-openai+ (get (bedrock/->mantle-openai-body (openai/openai-request-body opts)) :max_output_tokens omitted)++ :openrouter (get (openrouter/openrouter-request-body opts) :max_tokens omitted)+ :zai (get (zai/zai-request-body opts) :max_tokens omitted)+ :moonshot (get (moonshot/moonshot-request-body opts) :max_tokens omitted)+ :mistral (get (mistral/mistral-request-body opts) :max_tokens omitted)+ :vllm (get (vllm/vllm-request-body opts) :max_tokens omitted)+ ;; xAI sends the cap as max_completion_tokens+ :xai (get (xai/xai-request-body opts) :max_completion_tokens omitted)+ :gemini (get-in (stream-generate-content/request-body opts)+ [:generationConfig :maxOutputTokens] omitted)))))++(def ^:private expected-chat-caps+ "The cap each surface sends for each model it serves, keyed by surface and then by that surface's own id spelling.++ Written out literally rather than derived from the default, so a change to the default, or to which surfaces send+ none, fails here instead of drifting quietly. [[omitted]] means no cap is sent at all."+ {;; Anthropic Messages, direct: the catalog's own dated and undated spellings+ :anthropic {"claude-fable-5-1" 32000+ "claude-fable-5" 32000+ "claude-opus-5-5" 32000+ "claude-opus-5" 32000+ "claude-opus-4-8" 32000+ "claude-opus-4-7" 32000+ "claude-opus-4-6" 32000+ "claude-sonnet-5-5" 32000+ "claude-sonnet-5" 32000+ "claude-sonnet-4-6" 32000+ "claude-opus-4-5-20251101" 32000+ "claude-sonnet-4-5-20250929" 32000+ "claude-haiku-4-5-20251001" 32000+ "claude-opus-4-1-20250805" 32000+ ;; an id no catalog names, and an admin-named deployment: the Messages API needs a cap, so+ ;; they get the same default rather than an omission+ "claude-fable-6" 32000+ "my-deployment-3" 32000}+ ;; OpenAI Responses, direct: no default cap is sent (see openai/openai-request-body)+ :openai {"gpt-6-astra" omitted+ "gpt-5.6-sol" omitted+ "gpt-5.6-terra" omitted+ "gpt-5.6-luna" omitted+ "gpt-5.5" omitted+ "gpt-5.5-pro" omitted+ "gpt-5.4" omitted+ "gpt-5.4-pro" omitted+ "gpt-5.4-mini" omitted+ "gpt-4o" omitted}+ ;; the managed proxy: resolve-model-ref hands the adapter the bare model+ :managed {"claude-sonnet-4-6" 32000}+ ;; Bedrock: vendor-prefixed+ :bedrock-anthropic {"anthropic.claude-fable-5-1" 32000+ "anthropic.claude-fable-5" 32000+ "anthropic.claude-opus-5-5" 32000+ "anthropic.claude-opus-5" 32000+ "anthropic.claude-opus-4-8" 32000+ "anthropic.claude-opus-4-7" 32000+ "anthropic.claude-sonnet-5-5" 32000+ "anthropic.claude-sonnet-5" 32000+ "anthropic.claude-haiku-4-5" 32000}+ :bedrock-openai {"openai.gpt-6-astra" 32000+ "openai.gpt-5.4" 32000+ "openai.gpt-5.4-2026-03-05" 32000+ "openai.gpt-5.5" 32000+ "openai.gpt-5.5-2026-04-23" 32000}+ ;; Azure: the bare deployment name, dateless, and cased however the admin named it+ :azure-anthropic {"claude-fable-5-1" 32000+ "claude-fable-5" 32000+ "claude-opus-5-5" 32000+ "claude-opus-5" 32000+ "claude-opus-4-8" 32000+ "claude-opus-4-7" 32000+ "claude-opus-4-6" 32000+ "claude-sonnet-5-5" 32000+ "claude-sonnet-5" 32000+ "claude-sonnet-4-6" 32000+ "claude-opus-4-5" 32000+ "claude-sonnet-4-5" 32000+ "claude-haiku-4-5" 32000+ "claude-opus-4-1" 32000+ "Claude-Opus-4-8" 32000+ "Claude-Haiku-4-5" 32000}+ :azure-openai {"gpt-6-astra" omitted+ "gpt-5.6-sol" omitted+ "gpt-5.6-terra" omitted+ "gpt-5.6-luna" omitted+ "gpt-5.6" omitted+ "gpt-5.5-pro" omitted+ "gpt-5.5" omitted+ "gpt-5.4-pro" omitted+ "gpt-5.4-mini" omitted+ "gpt-5.4-nano" omitted+ "gpt-5.4" omitted+ "GPT-5.5" omitted}+ ;; Vertex: partner ids, dated with `@`+ :vertex-claude {"claude-fable-5-1" 32000+ "claude-fable-5" 32000+ "claude-opus-5-5" 32000+ "claude-opus-5" 32000+ "claude-opus-4-6" 32000+ "claude-sonnet-5-5" 32000+ "claude-sonnet-5" 32000+ "claude-sonnet-4-6" 32000+ "claude-haiku-4-5@20251001" 32000}+ :gemini {"google/gemini-3.5-flash" 32000+ "google/gemini-3.6-flash" 32000+ "google/gemini-3.7-flash" 32000+ "google/gemini-2.0-flash" 32000}+ :deepseek {"deepseek-v4-pro" 32000+ "deepseek-flash" 32000}+ ;; OpenRouter: dotted Claude versions, dated DeepSeek snapshots+ :openrouter {"anthropic/claude-fable-5.1" 32000+ "anthropic/claude-fable-5" 32000+ "anthropic/claude-opus-5" 32000+ "anthropic/claude-opus-4.8" 32000+ "anthropic/claude-opus-4.7" 32000+ "anthropic/claude-opus-4.6" 32000+ "anthropic/claude-sonnet-5" 32000+ "anthropic/claude-sonnet-4.6" 32000+ "anthropic/claude-opus-4.5" 32000+ "anthropic/claude-sonnet-4.5" 32000+ "anthropic/claude-haiku-4.5" 32000+ "anthropic/claude-opus-4.1" 32000+ "deepseek/deepseek-v4-pro" 32000+ "deepseek/deepseek-v4-pro-0813" 32000+ "deepseek/deepseek-v4-flash-0731" 32000+ "z-ai/glm-5.3" 32000+ "z-ai/glm-5.2" 32000+ "qwen/qwen3.8-max" 32000+ ;; the id qwen/qwen3.8-max resolves to on OpenRouter today+ "qwen/qwen3.8-max-0902" 32000+ ;; unlike on OpenAI direct and Azure, GPT models get the default here+ "openai/gpt-5.6-sol" 32000+ "openai/gpt-5.6-terra" 32000+ "openai/gpt-5.6-luna" 32000+ "openai/gpt-5.5" 32000+ "openai/gpt-5.5-pro" 32000+ "openai/gpt-5.4" 32000+ "openai/gpt-5.4-mini" 32000+ "openai/gpt-5.4-pro" 32000+ "mistralai/mistral-medium-3-5" 32000+ "moonshotai/kimi-k3" 32000+ ;; ids no catalog names get the default too+ "anthropic/claude-fable-6" 32000+ "openai/gpt-4o" 32000}+ :zai {"glm-5.3" 32000+ "glm-5.2" 32000+ "glm-4.7" 32000}+ :moonshot {"kimi-k3" 32000+ "kimi-k2.6" 32000+ "kimi-k2.7-code" 32000}+ :mistral {"mistral-medium-3-5" 32000+ "mistral-medium-latest" 32000}+ :xai {"grok-4.7" 32000+ "grok-4.3" 32000}+ ;; no default cap without a known context window, which the pure builder is not given+ ;; (see vllm/vllm-request-body)+ :vllm {"Qwen/Qwen3-32B" omitted}})++(deftest ^:parallel chat-cap-is-the-flat-default-test+ (testing "each surface sends the same default cap for every model, or deliberately none"+ (doseq [[surface models] expected-chat-caps+ [model expected] models]+ (testing (str surface " " model)+ (is (= expected (cap-for surface model)))))))++(deftest ^:parallel caller-cap-always-wins-test+ (testing "a task cap the caller names is sent verbatim, never replaced by the default"+ (doseq [[surface models] expected-chat-caps+ model (keys models)]+ (testing (str surface " " model)+ (is (= 512 (cap-for surface model {:max-tokens 512})))))))++(deftest ^:parallel forced-tool-call-without-a-caller-cap-gets-the-chat-cap-test+ (testing "the default cap clears the 2048 forced-tool-call floors, so the floor changes nothing"+ (are [surface model expected] (= expected (cap-for surface model {:schema {:type "object"}}))+ :openrouter "qwen/qwen3.8-max" 32000+ :openrouter "anthropic/claude-fable-5" 32000+ :openrouter "z-ai/glm-5.3" 32000+ :zai "glm-5.3" 32000+ :moonshot "kimi-k3" 32000+ :gemini "google/gemini-3.7-flash" 32000))+ (testing "vLLM with no known window still sends no cap: a floor raises a caller's cap, it never adds one"+ (is (= omitted (cap-for :vllm "Qwen/Qwen3-32B" {:schema {:type "object"}})))))++(defn- registry-models+ "The default and mini model the provider registry names for `type-name`, nils dropped: Azure has no default+ (its model is a deployment name), and the single-model types have no cheaper tier."+ [type-name]+ (keep identity [(llm.provider/default-model type-name) (llm.provider/mini-model type-name)]))++(defn- catalog-pairs+ "Every `[surface id]` the shipped catalogs offer, in the spelling that surface's builder receives.+ Bedrock and Azure each serve two wire families, so their ids are split by the prefix that selects one."+ []+ (concat+ (for [id (concat (keys claude/supported-models) (registry-models "anthropic"))] [:anthropic id])+ (for [id (concat (keys openai/supported-models) (registry-models "openai"))] [:openai id])+ (for [id (concat (keys deepseek/supported-models) (registry-models "deepseek"))] [:deepseek id])+ (for [id (concat (keys moonshot/supported-models) (registry-models "moonshot"))] [:moonshot id])+ (for [id (concat (keys mistral/supported-models) (registry-models "mistral"))] [:mistral id])+ (for [id (concat (keys zai/supported-models) (registry-models "zai"))] [:zai id])+ (for [id (concat (keys openrouter/supported-models) (registry-models "openrouter"))] [:openrouter id])+ (for [id (concat (keys xai/supported-models) (registry-models "xai"))] [:xai id])+ (for [id (concat (keys bedrock/supported-models) (registry-models "bedrock"))]+ [(if (str/starts-with? id "anthropic.") :bedrock-anthropic :bedrock-openai) id])+ ;; Azure lists no models; its context map is the closest thing to a catalog, and its keys are spelled the way+ ;; deployment names reach the builder.+ (for [id (keys @#'azure/model-context-windows)]+ [(if (str/starts-with? id "claude-") :azure-anthropic :azure-openai) id])+ (for [id (keys google.models/catalog)] [:gemini id])+ ;; Google's and the managed provider's catalogs are fixed in the registry. Google's Gemini ids reach the+ ;; builder whole; its Anthropic ids and the managed one lose the publisher segment on the way.+ (for [id (concat (map :id (llm.provider/fixed-models "google")) (registry-models "google"))]+ (if (str/starts-with? id "google/")+ [:gemini id]+ [:vertex-claude (llm.provider/model-ref->model id)]))+ (for [id (concat (map :id (llm.provider/fixed-models "metabase")) (registry-models "metabase"))]+ [:managed (llm.provider/model-ref->model id)])))++(deftest ^:parallel every-catalog-model-is-classified-test+ (testing "every model we ship has a deliberate expected cap, so a new catalog entry cannot slip in unclassified"+ ;; One-directional on purpose: expected-chat-caps also holds unknown ids and alternate spellings that no+ ;; catalog names.+ (let [classified (set (for [[surface models] expected-chat-caps+ model (keys models)]+ [surface model]))]+ (doseq [pair (catalog-pairs)]+ (testing (pr-str pair)+ (is (contains? classified pair)))))))diff --git a/test/metabase/metabot/self/claude_test.clj b/test/metabase/metabot/self/claude_test.cljindex 9c767af1706..d02a9ff6cc7 100644--- a/test/metabase/metabot/self/claude_test.clj+++ b/test/metabase/metabot/self/claude_test.clj@@ -678,23 +678,23 @@4096 {:schema schema :max-tokens 4096}512 {:max-tokens 512}))))-(deftest ^:parallel every-supported-model-has-a-ceiling-test- (doseq [[id {:keys [display-name max-tokens]}] @#'claude/supported-models]- (is (pos-int? max-tokens) id)+(deftest ^:parallel every-supported-model-has-a-display-name-test+ (doseq [[id {:keys [display-name]}] @#'claude/supported-models](is (seq display-name) id)))(deftest claude-max-tokens-test(mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key "sk-ant-test"](let [max-tokens #(:max_tokens (capture-claude-request-body!(merge {:input [{:role :user :content "hi"}]} %)))]- (are [opts tokens] (= tokens (max-tokens opts))- {:model "claude-opus-4-8"} 128000- {:model "claude-fable-5-1"} 128000- {:model "claude-haiku-4-5-20251001"} 64000- {:model "claude-opus-4-8" :max-tokens 32000} 32000- ;; Bedrock ids reach us vendor-prefixed- {:model "anthropic.claude-opus-4-8"} 128000- {:model "my-deployment-3"} @#'claude/default-max-tokens))))+ (testing "every model gets the same default cap, and a caller's own cap wins"+ (are [opts tokens] (= tokens (max-tokens opts))+ {:model "claude-opus-4-8"} 32000+ {:model "claude-fable-5-1"} 32000+ {:model "claude-haiku-4-5-20251001"} 32000+ {:model "claude-opus-4-8" :max-tokens 4096} 4096+ ;; Bedrock ids reach us vendor-prefixed+ {:model "anthropic.claude-opus-4-8"} 32000+ {:model "my-deployment-3"} 32000)))))(deftest claude-auto-cache-breakpoint-test(mt/with-temporary-setting-values [llm.settings/llm-anthropic-api-key "sk-ant-test"]diff --git a/test/metabase/metabot/self/deepseek_test.clj b/test/metabase/metabot/self/deepseek_test.cljindex 0a52f06e6c8..4e51339a794 100644--- a/test/metabase/metabot/self/deepseek_test.clj+++ b/test/metabase/metabot/self/deepseek_test.clj@@ -124,8 +124,8 @@:tools [(metabot.tu/get-time-tool)]})](testing "the model defaults to deepseek-v4-pro"(is (= "deepseek-v4-pro" (:model body))))- (testing "an unknown model id falls back to the shared max_tokens ceiling"- (is (= 64000 (:max_tokens body))))+ (testing "max_tokens is the Claude request body's default cap"+ (is (= 32000 (:max_tokens body))))(is (true? (:stream body)))(testing "the system prompt is sent as cache-marked content blocks"(is (=? [{:type "text" :text "You are Metabot." :cache_control {:type "ephemeral"}}]diff --git a/test/metabase/metabot/self/google/raw_predict_test.clj b/test/metabase/metabot/self/google/raw_predict_test.cljindex d30bd348c0a..684672528b6 100644--- a/test/metabase/metabot/self/google/raw_predict_test.clj+++ b/test/metabase/metabot/self/google/raw_predict_test.clj@@ -41,27 +41,16 @@:input [{:role :user :content "hi"}]}):speed)))))-(deftest ^:parallel request-body-dated-model-resolves-max-tokens-test- (testing "a model dated in the platform's `@` spelling resolves to the same max_tokens ceiling as its- direct-API `-` spelling, rather than falling back to the unknown-model default"- (is (= 32000- (:max_tokens (raw-predict/request-body "claude-opus-4-1@20250805"- {:input [{:role :user :content "hi"}]}))))))--(deftest ^:parallel request-body-bare-model-resolves-max-tokens-test- (testing "an undated model ID is already in the direct-API spelling and resolves as-is"- (is (= 128000- (:max_tokens (raw-predict/request-body "claude-sonnet-4-6"- {:input [{:role :user :content "hi"}]}))))))--(deftest ^:parallel request-body-unknown-model-gets-default-max-tokens-test- (testing "a model outside the Claude adapter's knowledge falls back to the safe default ceiling"- (is (= 64000- (:max_tokens (raw-predict/request-body "claude-not-a-real-model"- {:input [{:role :user :content "hi"}]}))))))+(deftest ^:parallel request-body-default-max-tokens-test+ (testing "every model gets the Claude request body's default max_tokens, dated, bare, or unknown"+ (are [model-id] (= 32000+ (:max_tokens (raw-predict/request-body model-id {:input [{:role :user :content "hi"}]})))+ "claude-opus-4-1@20250805"+ "claude-sonnet-4-6"+ "claude-not-a-real-model")))(deftest ^:parallel request-body-explicit-max-tokens-wins-test- (testing "an explicit :max-tokens overrides the model's ceiling"+ (testing "an explicit :max-tokens overrides the default"(is (= 4096(:max_tokens (raw-predict/request-body "claude-opus-4-1@20250805"{:input [{:role :user :content "hi"}]diff --git a/test/metabase/metabot/self/google/stream_generate_content_test.clj b/test/metabase/metabot/self/google/stream_generate_content_test.cljindex 649e9f49930..32cd6cec10f 100644--- a/test/metabase/metabot/self/google/stream_generate_content_test.clj+++ b/test/metabase/metabot/self/google/stream_generate_content_test.clj@@ -235,14 +235,16 @@(deftest ^:parallel request-body-system-instruction-test(testing "the system prompt becomes systemInstruction, outside :contents"(is (= {:systemInstruction {:parts [{:text "You are a helpful assistant."}]}- :contents [{:role "user" :parts [{:text "hi"}]}]}+ :contents [{:role "user" :parts [{:text "hi"}]}]+ :generationConfig {:maxOutputTokens 32000}}(sgc/request-body{:system "You are a helpful assistant.":input [{:role :user :content "hi"}]})))))(deftest ^:parallel request-body-no-system-message-test(testing "no systemInstruction is sent when system is not provided"- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]+ :generationConfig {:maxOutputTokens 32000}}(sgc/request-body {:input [{:role :user :content "hi"}]})))))(deftest ^:parallel request-body-generation-config-test@@ -254,9 +256,9 @@:temperature 0.2})))))(deftest ^:parallel request-body-generation-config-only-when-set-test- (testing "maxOutputTokens is omitted when the caller doesn't pass max-tokens, letting the model use its own limit"+ (testing "maxOutputTokens falls back to the default cap when the caller doesn't pass max-tokens"(is (= {:contents [{:role "user" :parts [{:text "hi"}]}]- :generationConfig {:temperature 0.2}}+ :generationConfig {:maxOutputTokens 32000 :temperature 0.2}}(sgc/request-body {:input [{:role :user :content "hi"}]:temperature 0.2}))))(testing "temperature is omitted when the caller doesn't pass one, letting the model use its own default"@@ -264,13 +266,15 @@:generationConfig {:maxOutputTokens 256}}(sgc/request-body {:input [{:role :user :content "hi"}]:max-tokens 256}))))- (testing "no generationConfig is sent when neither max-tokens nor temperature is provided"- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}+ (testing "generationConfig carries only the default cap when neither max-tokens nor temperature is provided"+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]+ :generationConfig {:maxOutputTokens 32000}}(sgc/request-body {:input [{:role :user :content "hi"}]})))))(deftest ^:parallel request-body-tool-choice-without-tools-test(testing "a tool_choice with no tools to choose from sends no toolConfig, which Gemini rejects on its own"- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]+ :generationConfig {:maxOutputTokens 32000}}(sgc/request-body {:input [{:role :user :content "hi"}]:tool_choice "required"})))))@@ -319,24 +323,26 @@(deftest ^:parallel request-body-structured-output-test(testing "a :schema forces a structured_output function call via mode ANY + allowedFunctionNames"(let [schema {:type "object" :properties {:sql {:type "string"}}}]- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]- :tools [{:functionDeclarations [{:name "structured_output"- :description "Output structured data"- :parametersJsonSchema schema}]}]- :toolConfig {:functionCallingConfig {:mode "ANY"- :allowedFunctionNames ["structured_output"]}}}+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]+ :generationConfig {:maxOutputTokens 32000}+ :tools [{:functionDeclarations [{:name "structured_output"+ :description "Output structured data"+ :parametersJsonSchema schema}]}]+ :toolConfig {:functionCallingConfig {:mode "ANY"+ :allowedFunctionNames ["structured_output"]}}}(sgc/request-body{:input [{:role :user :content "hi"}]:schema schema}))))))(deftest ^:parallel request-body-structured-output-replaces-tools-test(testing "a :schema replaces the caller's tools and tool_choice"- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]- :tools [{:functionDeclarations [{:name "structured_output"- :description "Output structured data"- :parametersJsonSchema {:type "object"}}]}]- :toolConfig {:functionCallingConfig {:mode "ANY"- :allowedFunctionNames ["structured_output"]}}}+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]+ :generationConfig {:maxOutputTokens 32000}+ :tools [{:functionDeclarations [{:name "structured_output"+ :description "Output structured data"+ :parametersJsonSchema {:type "object"}}]}]+ :toolConfig {:functionCallingConfig {:mode "ANY"+ :allowedFunctionNames ["structured_output"]}}}(sgc/request-body{:input [{:role :user :content "hi"}]:tools [{:tool-name "get_weather" :doc "Get the weather."@@ -361,15 +367,18 @@:reasoning? false:schema {:type "object"}}))))(testing ":reasoning? false sends no thinkingConfig, leaving the server default"- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]+ :generationConfig {:maxOutputTokens 32000}}(sgc/request-body {:model "google/gemini-3.6-flash":input [{:role :user :content "hi"}]:reasoning? false}))))(testing "an off-catalog or absent model gets no thinkingConfig at all"- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]+ :generationConfig {:maxOutputTokens 32000}}(sgc/request-body {:model "google/gemini-2.5-flash":input [{:role :user :content "hi"}]})))- (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]}+ (is (= {:contents [{:role "user" :parts [{:text "hi"}]}]+ :generationConfig {:maxOutputTokens 32000}}(sgc/request-body {:input [{:role :user :content "hi"}]})))))(deftest ^:parallel request-body-forced-tool-call-token-floor-test@@ -385,11 +394,16 @@:input [{:role :user :content "hi"}]:schema {:type "object"}:max-tokens 8000}))))- (testing "an uncapped structured call stays uncapped — the floor raises, it never introduces a cap"- (is (nil? (get-in (sgc/request-body {:model "google/gemini-3.7-flash"- :input [{:role :user :content "hi"}]- :schema {:type "object"}})- [:generationConfig :maxOutputTokens]))))+ (testing "an uncapped structured call takes the default cap, already above the floor"+ (is (= 32000 (get-in (sgc/request-body {:model "google/gemini-3.7-flash"+ :input [{:role :user :content "hi"}]+ :schema {:type "object"}})+ [:generationConfig :maxOutputTokens]))))+ (testing "an off-catalog model's uncapped structured call takes the same default cap"+ (is (= 32000 (get-in (sgc/request-body {:model "google/gemini-2.5-flash"+ :input [{:role :user :content "hi"}]+ :schema {:type "object"}})+ [:generationConfig :maxOutputTokens]))))(testing "an unforced call keeps the caller's cap"(is (=? {:generationConfig {:maxOutputTokens 512}}(sgc/request-body {:model "google/gemini-3.7-flash"@@ -416,6 +430,24 @@(is (=? {:generationConfig {:maxOutputTokens 512}}(body-for "auto"))))))+(deftest ^:parallel request-body-default-max-tokens-test+ (let [input [{:role :user :content "hi"}]]+ (testing "an uncapped call gets the default cap, on and off the catalog"+ (are [model] (= 32000 (get-in (sgc/request-body {:model model :input input})+ [:generationConfig :maxOutputTokens]))+ "google/gemini-3.5-flash"+ "google/gemini-3.6-flash"+ "google/gemini-3.7-flash"+ "google/gemini-2.5-flash"))+ (testing "a request naming no model at all gets it too"+ (is (= 32000 (get-in (sgc/request-body {:input input})+ [:generationConfig :maxOutputTokens]))))+ (testing "the caller's own task cap wins over the default"+ (is (= 512 (get-in (sgc/request-body {:model "google/gemini-3.7-flash"+ :input input+ :max-tokens 512})+ [:generationConfig :maxOutputTokens]))))))+;;; ──────────────────────────────────────────────────────────────────;;; Streaming event conversion tests.;;; ──────────────────────────────────────────────────────────────────diff --git a/test/metabase/metabot/self/google_test.clj b/test/metabase/metabot/self/google_test.cljindex ebaf1dede8b..3e7d88cbd4f 100644--- a/test/metabase/metabot/self/google_test.clj+++ b/test/metabase/metabot/self/google_test.clj@@ -161,7 +161,8 @@:headers {"Content-Type" "application/json"}}req))(is (= {:contents [{:role "user" :parts [{:text "hi"}]}]- :generationConfig {:thinkingConfig {:includeThoughts true}}}+ :generationConfig {:maxOutputTokens 32000+ :thinkingConfig {:includeThoughts true}}}(json/decode+kw (:body req)))))))))(deftest google-raw-regional-location-host-test@@ -401,8 +402,7 @@"the URL names the model; a model in the body is rejected by the platform"))))))(deftest google-raw-anthropic-max-tokens-test- (testing "the Anthropic model whitelist supplies max_tokens for a bare current-generation ID, and the @-versioned- spelling of a dated ID resolves to the same entry the direct Anthropic adapter uses"+ (testing "an Anthropic model gets the default max_tokens, whether its ID is bare or @-versioned"(mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"llm.settings/llm-google-service-account-key nilllm.settings/llm-google-project-id "my-project"@@ -411,11 +411,11 @@self.core/reducible-with-api-errors (fn [r _ _] r)debug/capture-stream (fn [r _] r)http/request (fn [req] {:body req})]- (doseq [[model max-tokens] {"anthropic/claude-sonnet-4-6" 128000- "anthropic/claude-fable-5" 128000- "anthropic/claude-haiku-4-5@20251001" 64000}]+ (doseq [model ["anthropic/claude-sonnet-4-6"+ "anthropic/claude-fable-5"+ "anthropic/claude-haiku-4-5@20251001"]](testing model- (is (= max-tokens+ (is (= 32000(:max_tokens (json/decode+kw (:body (google-raw {:model model:input [{:role :user :content "hi"}]}))))))))))))@@ -1322,12 +1322,14 @@:as :stream}post-req))(testing "the body is a Chat Completions request with an empty model, since the endpoint serves one"- (is (=? {:model ""- :stream true- :stream_options {:include_usage true}- :messages [{:role "user" :content "hi"}]- :max_tokens pos-int?}- (json/decode+kw (:body post-req))))))))+ (let [body (json/decode+kw (:body post-req))]+ (is (=? {:model ""+ :stream true+ :stream_options {:include_usage true}+ :messages [{:role "user" :content "hi"}]}+ body))+ (testing "and, like vLLM chat with an unknown context window, no output cap"+ (is (not (contains? body :max_tokens)))))))))(deftest google-raw-dedicated-endpoint-host-test(testing "a dedicated endpoint is served on the DNS name its resource reports, in either spelling Google uses, also when the base URL names Google's host for the location"diff --git a/test/metabase/metabot/self/moonshot_test.clj b/test/metabase/metabot/self/moonshot_test.cljindex 47fda381a50..3a2623246f5 100644--- a/test/metabase/metabot/self/moonshot_test.clj+++ b/test/metabase/metabot/self/moonshot_test.clj@@ -123,7 +123,8 @@2048 {:model "kimi-k3" :input [{:role :user :content "hi"}] :max-tokens 512:tools [(metabot.tu/get-time-tool)] :tool_choice "required"}4096 {:model "kimi-k3" :input [{:role :user :content "hi"}] :schema schema :max-tokens 4096}- nil {:model "kimi-k3" :input [{:role :user :content "hi"}] :schema schema}+ ;; uncapped by the caller, so the default cap applies — already above the floor+ 32000 {:model "kimi-k3" :input [{:role :user :content "hi"}] :schema schema}512 {:model "kimi-k3" :input [{:role :user :content "hi"}] :max-tokens 512}512 {:model "kimi-k2.6" :input [{:role :user :content "hi"}] :schema schema :max-tokens 512}512 {:model "kimi-k2.6" :input [{:role :user :content "hi"}] :max-tokens 512diff --git a/test/metabase/metabot/self/openrouter_test.clj b/test/metabase/metabot/self/openrouter_test.cljindex bc7329df5d8..f6bc8521a8e 100644--- a/test/metabase/metabot/self/openrouter_test.clj+++ b/test/metabase/metabot/self/openrouter_test.clj@@ -244,10 +244,11 @@;; safety net — its reasoning cannot be disabled and bills against the same budget as the;; tool call; theory vs practice in openrouter/forced-tool-call-token-floor(are [expected opts] (= expected (:max_tokens (body-for (assoc opts :model "qwen/qwen3.8-max-0902"))))- 2048 {:schema {:type "object"} :max-tokens 512}- 4096 {:schema {:type "object"} :max-tokens 4096}- nil {:schema {:type "object"}}- 512 {:max-tokens 512}))+ 2048 {:schema {:type "object"} :max-tokens 512}+ 4096 {:schema {:type "object"} :max-tokens 4096}+ ;; uncapped by the caller, so the default cap applies — already far above the floor+ 32000 {:schema {:type "object"}}+ 512 {:max-tokens 512}))(testing "a non-mandatory model keeps its cap — it got the disable instead"(let [body (body-for {:model "anthropic/claude-sonnet-4.6" :schema {:type "object"} :max-tokens 512})](is (= 512 (:max_tokens body)))diff --git a/test/metabase/metabot/self/vllm_test.clj b/test/metabase/metabot/self/vllm_test.cljindex d8a2f6e37b2..ce0f6a752da 100644--- a/test/metabase/metabot/self/vllm_test.clj+++ b/test/metabase/metabot/self/vllm_test.clj@@ -37,14 +37,56 @@;;; vllm-request-body;;; ──────────────────────────────────────────────────────────────────-(deftest ^:parallel request-body-applies-default-max-tokens-test- (testing "an explicit max_tokens is always sent — without one vLLM falls back to the whole remaining context window"- (is (= (llm.settings/llm-max-tokens)- (:max_tokens (vllm/vllm-request-body {:model "vllm-test"- :input [{:role :user :content "hi"}]}))))))+(deftest ^:parallel request-body-sends-no-default-max-tokens-test+ (testing "no max_tokens is sent without a caller cap or a known window — vLLM would subtract one from the+ admissible prompt, and nothing says how much room is left"+ (is (not (contains? (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]})+ :max_tokens)))))++(defn- cap-for-window+ "The `max_tokens` [[vllm/vllm-request-body]] sends for `opts` on a `window`-token context window, or+ `::omitted`."+ [opts window]+ (get (vllm/vllm-request-body (merge {:model "vllm-test" :input [{:role :user :content "hi"}]} opts) window)+ :max_tokens+ ::omitted))++(defn- long-input+ "A one-message input of `n` copies of `s`."+ [s n]+ [{:role :user :content (apply str (repeat n s))}])++(deftest ^:parallel request-body-sends-the-chat-cap-when-it-fits-the-window-test+ (testing "with a known window, the agent loop gets the flat chat cap"+ (is (= self.core/chat-max-output-tokens (cap-for-window {} 131072)))))++(deftest ^:parallel request-body-drops-a-cap-that-does-not-fit-the-window-test+ (testing "the chat cap alone is larger than the smallest window preflight accepts, so it is not sent"+ (is (= ::omitted (cap-for-window {} 16384))))+ (testing "the prompt counts: a cap that fits beside a short prompt is dropped beside a long one"+ (is (= 32000 (cap-for-window {} 65536)))+ (is (= ::omitted (cap-for-window {:input (long-input "x" 70000)} 65536))))+ (testing "a caller's cap is dropped the same way rather than lowered"+ (is (= ::omitted (cap-for-window {:input (long-input "x" 70000) :max-tokens 512} 32768))))+ (testing "digits count one token each, as Qwen3 tokenizes them"+ ;; Qwen3 gives each digit its own token, so 40000 digits are 40000 tokens and 32000 does not fit in+ ;; 65536; at 2 bytes per token the estimate was about 21000 and sent a cap that vLLM rejects+ (is (= ::omitted (cap-for-window {:input (long-input "7" 40000)} 65536))))+ (testing "a non-ASCII prompt is counted in bytes, not characters"+ ;; 25000 CJK characters are 75000 UTF-8 bytes, about 76000 estimated tokens, so 32000 no longer fits;+ ;; counted as characters they would leave room for it+ (is (= ::omitted (cap-for-window {:input (long-input "表" 25000)} 65536)))))++(deftest ^:parallel request-body-reasoning-floor-on-the-smallest-window-test+ (testing "the 16384 reasoning floor fills the whole 16384 window preflight accepts, so a raised cap is dropped+ rather than sent into a 400 — even on a 512-token title call"+ (is (= ::omitted (cap-for-window {:max-tokens 512 :credentials reasoning-credentials} 16384))))+ (testing "and on a window with room for it, the floor still applies"+ (is (= 16384 (cap-for-window {:max-tokens 512 :credentials reasoning-credentials} 32768)))))(deftest ^:parallel request-body-caller-max-tokens-wins-test- (testing "a caller-supplied max-tokens is not overridden by the default"+ (testing "a caller-supplied max-tokens is sent as-is"(is (= 128(:max_tokens (vllm/vllm-request-body {:model "vllm-test":input [{:role :user :content "hi"}]@@ -64,7 +106,13 @@:input [{:role :user :content "hi"}]:tools [(metabot.tu/get-time-tool)]:tool_choice "required"- :max-tokens 128})))))))+ :max-tokens 128}))))))+ (testing "a forced tool call with no caller cap and no known window sends none — a floor raises a cap,+ it never adds one"+ (is (not (contains? (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :schema {:type "object"}})+ :max_tokens)))))(deftest ^:parallel request-body-floor-never-lowers-a-ceiling-test(testing "a caller-supplied ceiling above the floor is left alone"@@ -84,17 +132,23 @@:max-tokens 128}))))))(deftest ^:parallel request-body-raises-max-tokens-for-a-reasoning-model-test- (testing "the agent path forces nothing and supplies no ceiling, so a reasoning model would otherwise get- the shared 4096 default for thinking, answer, and tool call combined"+ (testing "a reasoning model's caller cap is raised, since thinking, answer, and tool call share one budget"(is (= 16384(:max_tokens (vllm/vllm-request-body {:model "vllm-test":input [{:role :user :content "hi"}]+ :max-tokens 128:credentials reasoning-credentials}))))- (testing "and a model the probe found does not reason keeps the default"- (is (= (llm.settings/llm-max-tokens)+ (testing "and a model the probe found does not reason keeps the caller's cap"+ (is (= 128(:max_tokens (vllm/vllm-request-body {:model "vllm-test":input [{:role :user :content "hi"}]- :credentials credentials})))))))+ :max-tokens 128+ :credentials credentials}))))))+ (testing "with no known window the agent path supplies no cap, so a reasoning model is sent none either"+ (is (not (contains? (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials reasoning-credentials})+ :max_tokens)))))(deftest ^:parallel request-body-reasoning-floor-outranks-the-forced-tool-call-floor-test(testing "a reasoning model has to clear its thinking before the forced call, so the higher floor wins"@@ -572,6 +626,103 @@:input [{:role :user :content "hi"}]:credentials credentials}))))))+;;; ──────────────────────────────────────────────────────────────────+;;; Context-window lookup+;;; ──────────────────────────────────────────────────────────────────++(defn- fresh-credentials+ "Credentials for a server no other test has looked up, so the context-window cache starts empty for it."+ []+ {:base-url (str "http://vllm-" (random-uuid) ".internal:8000/v1")})++(defn- chat-with-catalog!+ "Run `vllm-raw` `n` times against a stub whose `/models` answers with `(models-response req)`. Return the+ `/models` requests it saw and the last chat body it was sent."+ [credentials models-response & {:keys [n] :or {n 1}}]+ (let [lookups (atom [])+ chat-body (atom nil)]+ (with-redefs [self.core/sse-reducible (fn [_] (reify clojure.lang.IReduceInit+ (reduce [_ _rf init] init)))+ debug/capture-stream (fn [r _] r)+ http/request (fn [{:keys [url body] :as req}]+ (if (re-find #"/models$" (str url))+ (do (swap! lookups conj req)+ (models-response req))+ (do (reset! chat-body (json/decode+kw body))+ {:body nil})))]+ (dotimes [_ n]+ (vllm/vllm-raw {:model "vllm-test" :input [{:role :user :content "hi"}] :credentials credentials})))+ {:lookups @lookups :chat-body @chat-body}))++(defn- catalog+ "A `/models` stub answering with `entries`."+ [& entries]+ (constantly {:status 200 :body {:data (vec entries)}}))++(deftest vllm-raw-sizes-max-tokens-from-the-served-window-test+ (testing "the window is read from the request's own model entry, not the first one listed"+ (let [{:keys [lookups chat-body]} (chat-with-catalog! (fresh-credentials)+ (catalog {:id "other" :max_model_len 16384}+ {:id "vllm-test" :max_model_len 131072}))]+ (is (= 32000 (:max_tokens chat-body)))+ (testing "with a short timeout, since the chat request waits for the lookup"+ (is (=? [{:method :get :socket-timeout 5000 :connection-timeout 5000}] lookups)))))+ (testing "a served window too small for the cap sends none"+ (is (not (contains? (:chat-body (chat-with-catalog! (fresh-credentials)+ (catalog {:id "vllm-test" :max_model_len 16384})))+ :max_tokens)))))++(deftest vllm-raw-treats-an-unreported-window-as-unknown-test+ (testing "no cap without a window: the model is not listed, or its entry has no `max_model_len`"+ (doseq [models-response [(catalog {:id "other" :max_model_len 131072})+ (catalog {:id "vllm-test"})+ (catalog {:id "vllm-test" :max_model_len "131072"})]]+ (is (not (contains? (:chat-body (chat-with-catalog! (fresh-credentials) models-response)) :max_tokens)))))+ (testing "a failed lookup is not an error: the chat request is still sent, with no cap"+ (doseq [failure [(SocketTimeoutException. "Read timed out")+ (ConnectException. "Connection refused")+ (ex-info "clj-http: status 404" {:status 404 :body "Not Found"})]]+ (let [{:keys [chat-body]} (chat-with-catalog! (fresh-credentials) (fn [_] (throw failure)))]+ (is (= "vllm-test" (:model chat-body)))+ (is (not (contains? chat-body :max_tokens)))))))++(deftest vllm-raw-caches-the-served-window-test+ (testing "one lookup serves the following requests to the same server and model"+ (let [{:keys [lookups chat-body]} (chat-with-catalog! (fresh-credentials)+ (catalog {:id "vllm-test" :max_model_len 131072})+ :n 3)]+ (is (= 1 (count lookups)))+ (is (= 32000 (:max_tokens chat-body)))))+ (testing "a failure is cached too, so a server that cannot answer does not slow every request"+ (is (= 1 (count (:lookups (chat-with-catalog! (fresh-credentials)+ (fn [_] (throw (SocketTimeoutException. "Read timed out")))+ :n 3)))))))++(defn- lookups-at!+ "The number of `/models` lookups one `vllm-raw` call makes at clock time `t`."+ [clock t credentials models-response]+ (reset! clock t)+ (count (:lookups (chat-with-catalog! credentials models-response))))++(deftest vllm-raw-keeps-an-answered-lookup-longer-than-a-failed-one-test+ (let [clock (atom 0)+ answered-ttl @#'vllm/answered-lookup-ttl-ms+ failed-ttl @#'vllm/failed-lookup-ttl-ms]+ (mt/with-dynamic-fn-redefs [vllm/now-ms (fn [] @clock)]+ (testing "an answer is reused until the long TTL runs out, with or without a window"+ (doseq [response [(catalog {:id "vllm-test" :max_model_len 131072})+ (catalog {:id "vllm-test"})]]+ (let [creds (fresh-credentials)]+ (is (= [1 0 1] [(lookups-at! clock 0 creds response)+ (lookups-at! clock (inc failed-ttl) creds response)+ (lookups-at! clock answered-ttl creds response)])))))+ (testing "a failed lookup is tried again after the short TTL"+ (let [creds (fresh-credentials)+ response (fn [_] (throw (SocketTimeoutException. "Read timed out")))]+ (is (= [1 0 1] [(lookups-at! clock 0 creds response)+ (lookups-at! clock (dec failed-ttl) creds response)+ (lookups-at! clock failed-ttl creds response)])))))))+;;; ──────────────────────────────────────────────────────────────────;;; list-models;;; ──────────────────────────────────────────────────────────────────diff --git a/test/metabase/metabot/self/xai_test.clj b/test/metabase/metabot/self/xai_test.cljindex aa6abd0365f..9bcd7de9dad 100644--- a/test/metabase/metabot/self/xai_test.clj+++ b/test/metabase/metabot/self/xai_test.clj@@ -36,6 +36,8 @@(let [body (xai/xai-request-body {:input input :max-tokens 512})](is (= 512 (:max_completion_tokens body)))(is (not (contains? body :max_tokens)))))+ (testing "a caller with no cap gets the chat cap, not xAI's 128,000 default"+ (is (= 32000 (:max_completion_tokens (xai/xai-request-body {:input input})))))(testing "chat keeps each model's default reasoning effort"(is (not (contains? (xai/xai-request-body {:input input :tools [(metabot.tu/get-time-tool)]}):reasoning_effort))))diff --git a/test/metabase/metabot/self/zai_test.clj b/test/metabase/metabot/self/zai_test.cljindex ec420686e6a..832e9d82abc 100644--- a/test/metabase/metabot/self/zai_test.clj+++ b/test/metabase/metabot/self/zai_test.clj@@ -105,8 +105,7 @@(testing "thinking-only models reject the disable (error 1210): no directive, reasoning_effort instead"(let [body (zai/zai-request-body {:model "glm-5.3" :input input})](is (not (contains? body :thinking)))- (is (= "max" (:reasoning_effort body)))- (is (not (contains? body :max_tokens)))))+ (is (= "max" (:reasoning_effort body)))))(testing "a schema drops the effort to low and raises a title-sized cap to the floor"(is (=? {:reasoning_effort "low":max_tokens 2048}@@ -140,6 +139,22 @@(is (not (contains? body :reasoning_effort)))(is (= 512 (:max_tokens body)))))))+(deftest ^:parallel request-body-default-max-tokens-test+ (let [input [{:role :user :content "hi"}]]+ (testing "an uncapped call gets the default cap, whatever the model"+ (are [model] (= 32000 (:max_tokens (zai/zai-request-body {:model model :input input})))+ "glm-5.3"+ "glm-5.2"+ "glm-4.7"))+ (testing "the caller's own task cap wins over the default"+ (is (= 512 (:max_tokens (zai/zai-request-body {:model "glm-5.3"+ :input input+ :max-tokens 512})))))+ (testing "a forced call takes the default, which is already above the floor"+ (is (= 32000 (:max_tokens (zai/zai-request-body {:model "glm-5.3"+ :input input+ :schema {:type "object"}})))))))+(deftest ^:parallel reasoning-model?-test(are [model expected] (= expected (zai/reasoning-model? model))"glm-5.2" truediff --git a/test/metabase/settings/models/setting_test.clj b/test/metabase/settings/models/setting_test.cljindex 4653754c0ad..519e78a4217 100644--- a/test/metabase/settings/models/setting_test.clj+++ b/test/metabase/settings/models/setting_test.clj@@ -1371,6 +1371,10 @@(is (= "Setting name 'retired-setting' is retired; use a different name instead"(ex-message e))))))))+(deftest ^:parallel llm-max-tokens-is-retired-test+ (testing "the removed llm-max-tokens setting's name cannot be reused by a new setting"+ (is (contains? @#'setting/retired-setting-names "llm-max-tokens"))))+(deftest duplicated-setting-name(testing "can re-register a setting in the same ns (redefining or reloading ns)"(is (defsetting foo (deferred-tru "A testing setting") :visibility :public :encryption :when-encryption-key-set))