Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[BOT-1930] Self hosted vLLM metabot support (#79437)

Tyler Plude8661fd452421
diff --git a/frontend/src/metabase-types/api/llm.ts b/frontend/src/metabase-types/api/llm.ts
index 0b77b151b88..3fea04e89e8 100644
--- a/frontend/src/metabase-types/api/llm.ts
+++ b/frontend/src/metabase-types/api/llm.ts
@@ -43,6 +43,7 @@ export type LlmProviderTypeName =
| "google"
| "azure"
| "bedrock"
+ | "vllm"
| "metabase";
export type LlmProviderFieldType =
diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.ts
index a2546644b49..5af7fbc9a17 100644
--- a/frontend/src/metabase-types/api/settings.ts
+++ b/frontend/src/metabase-types/api/settings.ts
@@ -542,6 +542,8 @@ interface SettingsManagerSettings {
"llm-bedrock-secret-access-key"?: string | null;
"llm-bedrock-region"?: string | null;
"llm-bedrock-session-token"?: string | null;
+ "llm-vllm-api-base-url"?: string | null;
+ "llm-vllm-api-key"?: string | null;
"openai-api-key": string | null;
"openai-available-models"?: OpenAiModel[];
"openai-model": string | null;
diff --git a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.tsx b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.tsx
index 641a20f52bb..ad9ad48cec7 100644
--- a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.tsx
+++ b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.tsx
@@ -19,7 +19,8 @@ const GENERIC_PROVIDER_ICON = "ai";
// Every mark is inlined rather than loaded by URL: it lets the single-colour ones take the theme's text
// colour the way their brands publish them, and it keeps them out of the emitted-asset pipeline. Metabase
-// is null because it is drawn from its own logo component.
+// is null because it is drawn from its own logo component, and vLLM because it falls back to the generic
+// AI icon.
const PROVIDER_LOGOS: Record<
LlmProviderTypeName,
ComponentType<SVGProps<SVGSVGElement>> | null
@@ -33,6 +34,7 @@ const PROVIDER_LOGOS: Record<
moonshot: MoonshotMark,
google: GoogleMark,
azure: AzureMark,
+ vllm: null,
metabase: null,
};
diff --git a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.unit.spec.tsx b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.unit.spec.tsx
index ff40f74fc95..a05399ea6b0 100644
--- a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.unit.spec.tsx
+++ b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.unit.spec.tsx
@@ -39,6 +39,13 @@ describe("ProviderTypeIcon", () => {
expect(screen.getByRole("presentation")).toBeInTheDocument();
});
+ it("falls back to a generic icon for vLLM, which is a server rather than a vendor", () => {
+ renderWithProviders(<ProviderTypeIcon type="vllm" />);
+
+ expect(screen.queryByTestId("main-logo")).not.toBeInTheDocument();
+ expect(screen.queryByRole("presentation")).not.toBeInTheDocument();
+ });
+
it("falls back to a generic icon for a type the frontend does not know yet", () => {
// the registry is the backend's, so it can serve a type this union does not list yet;
// the cast reproduces that server response, which is unreachable through the type alone
diff --git a/src/metabase/llm/api/provider.clj b/src/metabase/llm/api/provider.clj
index 4a6eb8846e2..ab0285b93d2 100644
--- a/src/metabase/llm/api/provider.clj
+++ b/src/metabase/llm/api/provider.clj
@@ -153,8 +153,13 @@
A type that names its model in `:config` (Azure, whose deployments its listing endpoint does not return) makes
the call for the same reason, but the model it serves comes from the connection rather than from the empty list
- that comes back."
- [{:keys [type config]} config-override model]
+ that comes back.
+
+ `probe?` asks a type that can check more than its credentials to do so — vLLM exercises the tool calling and
+ structured output the agent loop depends on against the model it will run on. Only [[verify-credentials!]] sets
+ it: a probe generates, so it is far too slow for a plain listing. A probe reports back the model it exercised as
+ `:probed-model` and anything it determined about the connection as `:learned-config`, both passed through here."
+ [{:keys [type config]} config-override model probe?]
(let [fixed (llm.provider/fixed-models type)]
(if (llm.provider/managed-type? type)
{:models (vec fixed)}
@@ -162,12 +167,14 @@
configured-model (llm.provider/connection-model type config)
model (or model configured-model (:id (first fixed)))]
(try
- (let [listed (:models (metabot.self/list-models type (cond-> {:credentials config}
- model (assoc :model model))))]
- {:models (cond
- configured-model [{:id configured-model :display_name (last (str/split configured-model #"/"))}]
- fixed (vec fixed)
- :else (vec listed))})
+ (let [listed (metabot.self/list-models type (cond-> {:credentials config}
+ model (assoc :model model)
+ probe? (assoc :probe? true)))]
+ (merge (select-keys listed [:probed-model :learned-config])
+ {:models (cond
+ configured-model [{:id configured-model :display_name (last (str/split configured-model #"/"))}]
+ fixed (vec fixed)
+ :else (vec (:models listed)))}))
(catch clojure.lang.ExceptionInfo e
(if (provider-client-error? e)
{:models [] :error (.getMessage e)}
@@ -193,7 +200,7 @@
(merge {:key conn-key :name conn-name :type type}
(try
(cache.wrapped/lookup-or-miss models-cache (models-cache-key conn)
- (fn [_] (list-connection-models* conn nil nil)))
+ (fn [_] (list-connection-models* conn nil nil false)))
(catch Exception e
(log/warn e "Failed to list models for LLM provider connection" {:connection conn-key})
{:models [] :error (.getMessage e)}))))
@@ -224,18 +231,36 @@
(into {} (remove (fn [[_ v]] (str/blank? v))) config))
(defn- verify-credentials!
- "Confirm `config` can actually reach `conn`'s provider before anything is persisted, by listing its models.
- Throws a 400 carrying the provider's own message when the credentials are rejected.
+ "Confirm `config` can actually reach `conn`'s provider before anything is persisted, by listing its models — and,
+ for a type that probes more than its credentials, by exercising the model it will run on. Throws a 400 carrying
+ the provider's own message when the credentials are rejected.
+
+ Returns `{:probed-model :learned-config}`: the model a probe exercised, for a type with no default model to fall
+ back on, and whatever the probe determined about the connection, for the caller to store on it.
The listing that verified the credentials is seeded into [[models-cache]] under the connection as it will be
stored, so the model refetch the client fires right after saving is answered from it instead of round-tripping
to the provider a second time."
[conn config model]
(when-not (llm.provider/managed-type? (:type conn))
- (let [{:keys [error] :as listed} (list-connection-models* conn config model)]
+ (let [{:keys [error learned-config] :as listed} (list-connection-models* conn config model true)]
(when error
(throw (ex-info error {:status-code 400 :api-error true})))
- (swap! models-cache cache/miss (models-cache-key (assoc conn :config config)) listed))))
+ (swap! models-cache cache/miss
+ (models-cache-key (assoc conn :config (merge config learned-config)))
+ (select-keys listed [:models]))
+ (select-keys listed [:probed-model :learned-config]))))
+
+(defn- selected-model
+ "The model `llm-metabot-provider` names for `conn-key`, or nil when the selection points elsewhere.
+
+ What an edited connection is verified against when the client names no model: for a type that checks more than
+ its credentials, the model the connection is actually serving is a better probe target than whichever one the
+ provider happens to list first."
+ [conn-key]
+ (let [model-ref (metabot.settings/llm-metabot-provider)]
+ (when (= conn-key (llm.provider/model-ref->connection-key model-ref))
+ (llm.provider/model-ref->model model-ref))))
(defn- connection-model-ref
"The `connection-key/model` reference that points Metabot at `conn`: the model the connection's own config names
@@ -383,12 +408,15 @@
:name (or (not-empty name) (str (:label provider-type)))
:config config}]
(llm.provider/validate-config! type config)
- (verify-credentials! conn config model)
- (let [had-usable-model? (metabot-has-a-usable-model?)]
+ (let [{:keys [probed-model learned-config]} (verify-credentials! conn config model)
+ conn (update conn :config merge learned-config)
+ had-usable-model? (metabot-has-a-usable-model?)]
(llm.provider/set-connections! (conj (llm.provider/stored-connections) conn))
(when-not had-usable-model?
- (select-model-for-new-connection! conn model)))
- (connection-response (assoc conn :source :db)))))
+ ;; a type with no default model — vLLM, which serves whatever the operator loaded — starts on the model
+ ;; the probe exercised, so connecting one leaves the instance working rather than model-less
+ (select-model-for-new-connection! conn (or model probed-model)))
+ (connection-response (assoc conn :source :db))))))
(api.macros/defendpoint :put "/providers/:key"
:- connection-response-schema
@@ -420,10 +448,12 @@
;; what the connection will actually run on: the stored config with the environment layered back over it
effective (merge (:config merged) env-config)]
(llm.provider/validate-config! (:type merged) effective)
- (verify-credentials! merged effective model)
- (llm.provider/set-connections! (assoc stored idx merged))
- (follow-edited-connection-model! (assoc merged :config effective) model)
- (connection-response (assoc (merge live merged) :config effective))))
+ (let [{:keys [learned-config]} (verify-credentials! merged effective (or model (selected-model conn-key)))
+ merged (update merged :config merge learned-config)
+ effective (merge effective learned-config)]
+ (llm.provider/set-connections! (assoc stored idx merged))
+ (follow-edited-connection-model! (assoc merged :config effective) model)
+ (connection-response (assoc (merge live merged) :config effective)))))
(api.macros/defendpoint :delete "/providers/:key" :- :nil
"Delete a provider connection."
diff --git a/src/metabase/llm/provider.clj b/src/metabase/llm/provider.clj
index 53780bb12fc..fc24ebc53f4 100644
--- a/src/metabase/llm/provider.clj
+++ b/src/metabase/llm/provider.clj
@@ -265,6 +265,25 @@
:type :password
:advanced? true
:help (deferred-tru "Only needed for temporary credentials.")}]}
+ {:type "vllm"
+ :label (deferred-tru "vLLM")
+ ;; A vLLM server serves whatever the operator loaded it with, so there is no model to default to: the one a
+ ;; new connection starts on comes from the catalog that connecting fetches (see
+ ;; [[metabase.metabot.self.vllm/list-models]]).
+ :default-model nil
+ :fields [{:key :base-url
+ :normalize strip-trailing-slashes
+ :label (deferred-tru "API base URL")
+ :type :text
+ :required? true
+ :placeholder "http://vllm.internal:8000/v1"
+ :help (deferred-tru "Your server''s OpenAI-compatible API. It should end in /v1.")}
+ {:key :api-key
+ :label (deferred-tru "API key")
+ :type :password
+ ;; not required: a server started without --api-key takes no key, and a base URL on its own
+ ;; is a complete configuration
+ :help (deferred-tru "Only needed if you started your server with --api-key.")}]}
{:type "metabase"
:label (deferred-tru "Metabase AI service")
:managed? true
@@ -487,7 +506,12 @@
:settings {:access-key-id {:setting :llm-bedrock-access-key-id :credential? true}
:secret-access-key {:setting :llm-bedrock-secret-access-key :credential? true}
:session-token {:setting :llm-bedrock-session-token}
- :region {:setting :llm-bedrock-region}}}})
+ :region {:setting :llm-bedrock-region}}}
+ "vllm" {:type "vllm"
+ ;; the base URL is the credential here, unlike Azure's: a server started without --api-key takes
+ ;; no key, so the URL alone brings a usable connection into existence
+ :settings {:base-url {:setting :llm-vllm-api-base-url :credential? true}
+ :api-key {:setting :llm-vllm-api-key}}}})
(defn- env-supplied-fields
"The `:config` fields the environment supplies for one [[single-provider-settings]] group:
diff --git a/src/metabase/llm/settings.clj b/src/metabase/llm/settings.clj
index a323e6685b1..3ec6f6e5a7e 100644
--- a/src/metabase/llm/settings.clj
+++ b/src/metabase/llm/settings.clj
@@ -387,6 +387,34 @@
:setter (connection-field-setter :llm-azure-deployment-name)
:doc "Backed by the azure connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.")
+;;; --------------------------------------------------- vLLM ----------------------------------------------------
+
+(defsetting llm-vllm-api-base-url
+ (deferred-tru "The base URL of your vLLM server''s OpenAI-compatible API, e.g. `http://vllm.internal:8000/v1`.")
+ :encryption :no
+ :visibility :settings-manager
+ :export? false
+ :getter (connection-field-getter :llm-vllm-api-base-url)
+ :setter (connection-field-setter :llm-vllm-api-base-url)
+ :doc "Backed by the vllm connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.")
+
+(defsetting llm-vllm-api-key
+ (deferred-tru "The API key for your vLLM server. Only needed when the server was started with `--api-key`.")
+ :sensitive? true
+ :visibility :settings-manager
+ :export? false
+ :getter (connection-field-getter :llm-vllm-api-key)
+ :setter (connection-field-setter :llm-vllm-api-key)
+ :doc "Backed by the vllm connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.")
+
+(defsetting llm-vllm-request-timeout-ms
+ (deferred-tru "Socket timeout in milliseconds for requests to your vLLM server.")
+ ;; Self-hosted TTFT is bounded by the operator's hardware; the shared 60s default is too short.
+ :type :integer
+ :default 300000
+ :visibility :settings-manager
+ :export? false)
+
;;; ---------------------------------------------- Provider connections ------------------------------------------
;;; The per-provider credential settings above are read-only at runtime: they configure a connection only when set
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 885b6b74db6..c0d35685e65 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -24,6 +24,7 @@
[metabase.metabot.self.moonshot :as moonshot]
[metabase.metabot.self.openai :as openai]
[metabase.metabot.self.openrouter :as openrouter]
+ [metabase.metabot.self.vllm :as vllm]
[metabase.metabot.self.zai :as zai]
[metabase.metabot.settings :as metabot.settings]
[metabase.metabot.usage :as usage]
@@ -45,6 +46,7 @@
"moonshot" moonshot/moonshot
"openai" openai/openai
"openrouter" openrouter/openrouter
+ "vllm" vllm/vllm
"zai" zai/zai
(throw (ex-info (str "Unknown LLM provider: " provider)
{:provider provider}))))
@@ -60,6 +62,7 @@
"moonshot" moonshot/list-models
"openai" openai/list-models
"openrouter" openrouter/list-models
+ "vllm" vllm/list-models
"zai" zai/list-models
(throw (ex-info (str "Unknown LLM provider: " provider)
{:provider provider}))))
@@ -141,14 +144,21 @@
error: `rethrow-api-error!` rethrows e.g. a `Read timed out` as an
`ExceptionInfo` (with `:error-code :provider-request-failed` and no `:status`)
whose *cause* is the original `SocketTimeoutException`. Inspecting only the
- top-level exception would miss it and we'd never retry a transient timeout."
+ top-level exception would miss it and we'd never retry a transient timeout.
+
+ An adapter opts a specific failure out of that default by tagging its ex-data
+ `:retryable? false` (top-level only), which wins over both checks. Note that
+ omitting `:status` does not — the cause walk still matches."
[^Exception e]
- (boolean
- (or (retryable-status? (:status (ex-data e)))
- ;; Connection errors (e.g. under load, connection refused/reset), possibly
- ;; wrapped one or more levels deep by a provider adapter. Bounded to 10
- ;; levels to guard against a cyclic getCause chain.
- (some connection-error? (take 10 (take-while some? (iterate #(some-> ^Throwable % .getCause) e)))))))
+ (let [data (ex-data e)]
+ (if (false? (:retryable? data))
+ false
+ (boolean
+ (or (retryable-status? (:status data))
+ ;; Connection errors (e.g. under load, connection refused/reset), possibly
+ ;; wrapped one or more levels deep by a provider adapter. Bounded to 10
+ ;; levels to guard against a cyclic getCause chain.
+ (some connection-error? (take 10 (take-while some? (iterate #(some-> ^Throwable % .getCause) e)))))))))
(defn- parse-retry-after-header
"Extract retry-after seconds from response headers in ex-data, if present and ≤ 60s.
diff --git a/src/metabase/metabot/self/openai/chat_completions.clj b/src/metabase/metabot/self/openai/chat_completions.clj
index b57b6ebf0ad..30af804eb42 100644
--- a/src/metabase/metabot/self/openai/chat_completions.clj
+++ b/src/metabase/metabot/self/openai/chat_completions.clj
@@ -111,6 +111,15 @@
"function_call" "tool-calls"
"content_filter" "content-filter"})
+(defn- delta-reasoning
+ "Reasoning text carried by a Chat Completions delta or message, under either spelling. vLLM 0.26
+ emits `reasoning` and treats `reasoning_content` as its deprecated name; older builds, Z.AI, and
+ other OpenAI-compatible servers still emit the latter, and a self-hosted server's version is the
+ customer's choice."
+ [m]
+ (or (not-empty (:reasoning m))
+ (not-empty (:reasoning_content m))))
+
(defn chat-completions->aisdk-chunks-xf
"Translates Chat Completions streaming chunks into AI SDK v5 protocol chunks.
@@ -138,13 +147,21 @@
would lose their arguments, since neither the start branch (needs `:name`) nor
the argument-delta branch (needs no `:id`) would fire.
- Takes the dialect's `finish_reason` table, defaulting to OpenAI's [[stop-reasons]]."
+ Takes the dialect's `finish_reason` table, defaulting to OpenAI's [[stop-reasons]].
+
+ `opts` may carry `:forward-reasoning?`, which additionally translates reasoning
+ deltas (see [[delta-reasoning]]) into :reasoning-start / :reasoning-delta /
+ :reasoning-end. Opt-in, because whether a provider's reasoning renders at all is
+ a separate question (see `metabot.settings/llm-metabot-supports-reasoning?`) and
+ chunks nothing consumes only add stream volume."
([]
- (chat-completions->aisdk-chunks-xf stop-reasons))
+ (chat-completions->aisdk-chunks-xf stop-reasons nil))
([stop-reasons]
+ (chat-completions->aisdk-chunks-xf stop-reasons nil))
+ ([stop-reasons {:keys [forward-reasoning?]}]
(fn [rf]
- (let [current-type (volatile! nil) ;; :text | :function_call | nil
- current-id (volatile! nil) ;; active chunk id (text-id or tool call_id)
+ (let [current-type (volatile! nil) ;; :text | :reasoning | :function_call | nil
+ current-id (volatile! nil) ;; active chunk id (text-id, reasoning-id, or tool call_id)
message-id (volatile! nil)
model-name (volatile! nil)
payload (volatile! {}) ;; carried across start/delta/end, same as openai.clj
@@ -152,6 +169,7 @@
close! (fn [result]
(u/prog1 (rf result (merge {:type (case @current-type
:text :text-end
+ :reasoning :reasoning-end
:function_call :tool-input-available)}
@payload))
(vreset! current-type nil)
@@ -172,9 +190,11 @@
;; Empty-string content (common between tool calls) is ignored
;; to avoid spurious text blocks that would close open tools.
chunk-type (cond
- (not-empty (:content delta)) :text
- (some? tool-call) :function_call
- :else nil)
+ (not-empty (:content delta)) :text
+ (and forward-reasoning?
+ (delta-reasoning delta)) :reasoning
+ (some? tool-call) :function_call
+ :else nil)
;; For new tool calls, the id comes from the chunk; for deltas
;; on the same tool, we keep current-id.
chunk-id (or (:id tool-call) @current-id (core/mkid))]
@@ -204,6 +224,18 @@
(some? (:content delta))) (rf {:type :text-delta
:id @current-id
:delta (:content delta)})
+ ;; Start a new reasoning block
+ (and (= chunk-type :reasoning)
+ (not= @current-type :reasoning)) (-> (u/prog1
+ (let [rid (core/mkid)]
+ (vreset! current-type :reasoning)
+ (vreset! current-id rid)
+ (vreset! payload {:id rid})))
+ (rf (merge {:type :reasoning-start} @payload)))
+ ;; Reasoning delta
+ (= chunk-type :reasoning) (rf {:type :reasoning-delta
+ :id @current-id
+ :delta (delta-reasoning delta)})
;; Start a new tool call block
(and (= chunk-type :function_call)
(:id tool-call)
@@ -279,11 +311,15 @@
no `:status`: this isn't a credentials problem, and `metabase.metabot.api`'s `provider-client-error?`
renders any 4xx under the admin API-key field, which would attach the wrong message to the wrong input.
- A well-formed but empty `data` is a legitimate response — an account with no accessible models — and passes."
- [provider-name res]
- (let [data (get-in res [:body :data])]
- (when-not (sequential? data)
- (throw (ex-info (tru "{0} returned an unexpected model list response" provider-name)
- {:api-error true
- :error-code :malformed-model-catalog})))
- data))
+ A well-formed but empty `data` is a legitimate response — an account with no accessible models — and passes.
+
+ `:detail` is a sentence appended to the message, for a provider that has something more specific to say."
+ ([provider-name res] (models-catalog provider-name res nil))
+ ([provider-name res {:keys [detail]}]
+ (let [data (get-in res [:body :data])]
+ (when-not (sequential? data)
+ (throw (ex-info (cond-> (tru "{0} returned an unexpected model list response" provider-name)
+ detail (str ". " detail))
+ {:api-error true
+ :error-code :malformed-model-catalog})))
+ data)))
diff --git a/src/metabase/metabot/self/openrouter.clj b/src/metabase/metabot/self/openrouter.clj
index 6371fede208..768d7cbf42e 100644
--- a/src/metabase/metabot/self/openrouter.clj
+++ b/src/metabase/metabot/self/openrouter.clj
@@ -135,6 +135,28 @@
[model]
(str/starts-with? (str model) "anthropic/"))
+(defn- anthropic-current-gen?
+ "Current-generation Claude on OpenRouter: Fable, Opus >= 4.7, Sonnet >= 5."
+ [model]
+ (or (str/starts-with? model "anthropic/claude-fable")
+ (when-let [[_ family major minor] (re-find #"^anthropic/claude-(opus|sonnet)-(\d+)(?:\.(\d+))?" model)]
+ (let [major (parse-long major)
+ minor (or (some-> minor parse-long) 0)]
+ (case family
+ "opus" (or (> major 4) (and (= major 4) (>= minor 7)))
+ "sonnet" (>= major 5))))))
+
+(defn- model-supports-temperature?
+ "Whether an OpenRouter model id accepts an explicit `temperature`. Two families reject it: OpenAI's
+ GPT-5 and o-series, and current-generation Claude. Neither `openai.clj`'s nor `claude.clj`'s
+ predicate can be reused — both are keyed to their own provider's id shape, and would silently
+ return true for OpenRouter's `vendor/model` ids with dotted versions."
+ [model]
+ (let [model (str model)]
+ (not (or (str/starts-with? model "openai/gpt-5")
+ (re-find #"^openai/o\d" model)
+ (anthropic-current-gen? model)))))
+
(def ^:private required-tool-choice-unsupported-models
"Models that don't support `:tool_choice \"required\"`"
#{"qwen/qwen3.8-max"})
@@ -158,13 +180,20 @@
unlike claude.clj we don't put a separate breakpoint there.
Other models (OpenAI) keep the generic plain string system message: OpenAI prompt caching is automatic server-side
- and takes no request markup."
+ and takes no request markup.
+
+ `:temperature` is dropped for models that reject it (see [[model-supports-temperature?]]). Gating it in the shared
+ builder instead would apply these OpenRouter-specific rules to every Chat Completions adapter, including vLLM,
+ whose model names are customer-chosen free text."
[{:keys [model system] :as opts
:or {model "anthropic/claude-haiku-4.5"}} :- core/LLMRequestOpts]
(cond-> (chat-completions/request-body (assoc opts :model model))
(and system (anthropic-model? model))
(update-in [:messages 0 :content] claude/system->cached-content-blocks)
+ (not (model-supports-temperature? model))
+ (dissoc :temperature)
+
(not (supports-required-tool-choice? model))
required-tool-choice->auto))
diff --git a/src/metabase/metabot/self/vllm.clj b/src/metabase/metabot/self/vllm.clj
new file mode 100644
index 00000000000..c92f00a3113
--- /dev/null
+++ b/src/metabase/metabot/self/vllm.clj
@@ -0,0 +1,524 @@
+(ns metabase.metabot.self.vllm
+ "vLLM / Chat Completions adapter for self-hosted, OpenAI-compatible inference servers.
+
+ The base URL is required and the API key is optional, and there is no model whitelist — the
+ operator serves whatever they loaded, so [[preflight!]] exercises the agent-loop contract at
+ configuration time instead.
+
+ https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html"
+ (:require
+ [clojure.string :as str]
+ [metabase.llm.settings :as llm]
+ [metabase.metabot.self.core :as core]
+ [metabase.metabot.self.debug :as debug]
+ [metabase.metabot.self.openai.chat-completions :as chat-completions]
+ [metabase.util :as u]
+ [metabase.util.i18n :refer [tru]]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.o11y :refer [with-span]])
+ (:import
+ (java.io IOException)
+ (java.net SocketTimeoutException)
+ (java.util.concurrent ExecutionException)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private min-context-length
+ "Smallest `max_model_len` [[preflight!]] will accept. A product floor, not a measurement."
+ 16384)
+
+(defn- ai-proxy-unsupported-ex []
+ (ex-info (tru "AI proxy is not supported for vLLM")
+ {:api-error true
+ :error-code :proxy-unsupported}))
+
+(defn- missing-base-url-ex []
+ (ex-info (tru "No vLLM base URL is set")
+ {:api-error true
+ :error-code :base-url-missing}))
+
+(defn- missing-model-ex []
+ (ex-info (tru "No vLLM model is set")
+ {:api-error true
+ :error-code :model-missing}))
+
+(defn- vllm-error-msg
+ "Canonical, status-specific vLLM error message."
+ [res]
+ (let [status (long (:status res 0))]
+ (case status
+ 400 (tru "vLLM rejected the request — usually an unsupported schema, or a model that cannot compile the tool grammar")
+ 401 (tru "vLLM API key expired or invalid — check the key your server was started with via --api-key")
+ 404 (tru "vLLM API endpoint was not found — the base URL should end in /v1")
+ 429 (tru "The vLLM server''s request queue is full — reduce concurrent load, or restart it with a larger --max-num-seqs")
+ 500 (tru "vLLM returned an internal server error")
+ (tru "vLLM API error (HTTP {0})" status))))
+
+(def reasoning-config-key
+ "The connection `:config` key [[preflight!]]'s reasoning observation is recorded under. It is not an
+ admin-entered field: whether a served model reasons depends on the operator's `--reasoning-parser`
+ as much as on the model, so only the probe can answer it."
+ :model-reasoning)
+
+(defn reasoning-connection?
+ "Whether the connection carrying `credentials` was observed streaming its reasoning. A hand-written
+ `llm-providers` can hold a JSON boolean where the API stores the string it round-trips."
+ [credentials]
+ (let [recorded (get credentials reasoning-config-key)]
+ (or (true? recorded) (= "true" recorded))))
+
+(defn- vllm-auth
+ "Auth map for a vLLM request, built from the connection's credentials alone. The map is never nil, so
+ `core/resolve-auth` cannot reach its `missing-api-key-ex` branch — a keyless server is a complete
+ configuration."
+ [credentials ai-proxy?]
+ (when ai-proxy?
+ (throw (ai-proxy-unsupported-ex)))
+ (let [base-url (not-empty (:base-url credentials))
+ api-key (not-empty (:api-key credentials))]
+ (when-not base-url
+ (throw (missing-base-url-ex)))
+ (core/resolve-auth "vllm" "vLLM"
+ (cond-> {:url base-url}
+ api-key (assoc :headers {"Authorization" (str "Bearer " api-key)}))
+ ai-proxy?)))
+
+(defn- inference-timeouts
+ "Timeouts for a generation request."
+ []
+ {:socket-timeout (llm/llm-vllm-request-timeout-ms)
+ :connection-timeout (llm/llm-connection-timeout-ms)})
+
+(defn- control-timeouts
+ "Timeouts for the non-generating `/models` call, on the shared (much shorter) request budget."
+ []
+ {:socket-timeout (llm/llm-request-timeout-ms)
+ :connection-timeout (llm/llm-connection-timeout-ms)})
+
+(def ^:private probe-timeout-ceiling-ms
+ "Upper bound on a single preflight probe, which blocks the admin behind a spinner."
+ 120000)
+
+(defn- probe-timeouts
+ "Timeouts for a preflight probe. An operator who lowers `llm-vllm-request-timeout-ms` below the
+ ceiling gets their own value."
+ []
+ {:socket-timeout (min (llm/llm-vllm-request-timeout-ms) probe-timeout-ceiling-ms)
+ :connection-timeout (llm/llm-connection-timeout-ms)})
+
+;;; ----------------------------------------------- Transport errors ---------------------------------------------
+
+(defn- unreachable-ex
+ "The vLLM error for a non-timeout transport failure. `extra` is the caller's own ex-data tags."
+ [^IOException e base-url extra]
+ (ex-info (tru "Could not reach the vLLM server at {0}. Check that it is running and that the base URL is correct."
+ (str base-url))
+ (merge {:api-error true :error-code :vllm-unreachable} extra)
+ e))
+
+(defn- list-models-io-ex
+ "The vLLM error for a transport failure while fetching the model catalog — the request behind the
+ admin Connect button. Tagged `:status-code 400` so a mistyped base URL surfaces the message rather
+ than the 500 `core/rethrow-api-error!`'s untagged no-response branch would produce."
+ [^IOException e base-url]
+ (if (instance? SocketTimeoutException e)
+ (ex-info (tru "The vLLM server at {0} did not respond within {1}ms. Check that it is running and not overloaded."
+ (str base-url) (str (llm/llm-request-timeout-ms)))
+ {:api-error true
+ :status-code 400
+ :error-code :vllm-timeout}
+ e)
+ (unreachable-ex e base-url {:status-code 400})))
+
+;;; ------------------------------------------------ Model listing -----------------------------------------------
+
+(defn- list-all-models
+ "Fetch the served model catalog, which doubles as the credential round-trip behind the admin
+ Connect button.
+
+ A 2xx whose body is not a recognizable catalog fails closed via
+ [[chat-completions/models-catalog]], naming the base URL — the likeliest cause for the one provider
+ whose base URL the admin types."
+ [auth]
+ (try
+ (let [res (core/request auth (merge {:method :get
+ :url "/models"
+ :as :json
+ :headers {"Content-Type" "application/json"}}
+ (control-timeouts)))]
+ ;; The URL off `auth`, not the setting: a connect verifies request credentials before saving them.
+ (chat-completions/models-catalog
+ "vLLM" res
+ {:detail (tru "Check that {0} is a vLLM server''s OpenAI-compatible API — the base URL should end in /v1."
+ (str (:url auth)))}))
+ ;; Ordered ahead of the generic catch, which a non-2xx still reaches as an `ExceptionInfo`.
+ ;; `:as :json` also lands a 2xx whose body is not JSON here, via Jackson's `JsonParseException`.
+ (catch IOException e
+ (throw (list-models-io-ex e (:url auth))))
+ (catch Exception e
+ (core/rethrow-api-error! "vllm" vllm-error-msg e))))
+
+;;; -------------------------------------------------- Preflight -------------------------------------------------
+
+(def ^:private probe-tool
+ {:type "function"
+ :function {:name "record_table_name"
+ :description "Record the name of the table the user mentioned."
+ :parameters {:type "object"
+ :properties {:table_name {:type "string"
+ :description "The table name the user mentioned."}}
+ :required ["table_name"]
+ :additionalProperties false}}})
+
+(def ^:private probe-messages
+ [{:role "user" :content "Record the table name: orders"}])
+
+(def ^:private probe-max-tokens
+ "Generation ceiling for a preflight probe. High enough to clear a reasoning model's thinking, which
+ is billed against it: a probe that stops at `length` before the tool call looks identical to a
+ server that will not call tools at all."
+ 2048)
+
+(def ^:private forced-tool-call-token-floor
+ "Smallest `max_tokens` a forced tool call is given, regardless of what the caller asked for — below
+ it a reasoning model spends the budget thinking and emits no tool call. Equal to
+ [[probe-max-tokens]], which [[preflight!]] already proves the served model can clear."
+ probe-max-tokens)
+
+(def ^:private reasoning-model-token-floor
+ "Smallest `max_tokens` any request gets once [[preflight!]] has observed the served model reasoning.
+ Chat Completions bills thinking, answer, and tool call against one budget. Mirrors
+ `claude-request-body`."
+ 16384)
+
+(def ^:private default-temperature
+ "Sampling temperature for a caller that supplies none. vLLM's own default is 1.0, which is wrong for
+ the tool-calling and SQL-generation work the agent loop does. The hosted providers pick a sane
+ default server-side; a self-hosted server does not, so the adapter supplies one."
+ 0.3)
+
+(defn- preflight-ex
+ "A preflight failure, tagged so `metabase.metabot.api` surfaces the message verbatim, not as a 500."
+ [msg]
+ (ex-info msg {:api-error true
+ :status-code 400
+ :error-code :vllm-preflight-failed}))
+
+(defn- probe-chat!
+ "Run one non-streaming Chat Completions turn against `model` and return the first choice. The
+ `finish_reason` is part of the return value because a generation truncated at
+ [[probe-max-tokens]] and a server that will not call tools both produce empty `tool_calls`."
+ [auth model tool-choice]
+ (let [res (core/request auth (merge {:method :post
+ :url "/chat/completions"
+ :as :json
+ :headers {"Content-Type" "application/json"}
+ :body (json/encode {:model model
+ :messages probe-messages
+ :tools [probe-tool]
+ :tool_choice tool-choice
+ :temperature 0
+ :max_tokens probe-max-tokens})}
+ (probe-timeouts)))]
+ (get-in res [:body :choices 0])))
+
+(defn- check-context-budget!
+ [{:keys [id max_model_len]}]
+ (when (and max_model_len (< (long max_model_len) min-context-length))
+ (throw (preflight-ex
+ (tru "{0} is served with a {1} token context window, which is too small for Metabot — it needs at least {2}. Restart vLLM with a larger --max-model-len."
+ (str id) (str max_model_len) (str min-context-length))))))
+
+(defn- check-tool-calling!
+ "Check that the server was started with `--enable-auto-tool-choice` and a `--tool-call-parser` whose
+ sentinels match this model. A wrong-but-valid parser name errors at no layer — the call stays in
+ `content` as prose and Metabot chats without ever acting.
+
+ Returns whether the model emitted reasoning, the only signal anywhere that it is a reasoning model."
+ [auth model]
+ (let [{:keys [message finish_reason]} (probe-chat! auth model "auto")
+ content (str (:content message))
+ ;; `reasoning` since vLLM 0.26; `reasoning_content` is the deprecated spelling older builds
+ ;; and other OpenAI-compatible servers still use.
+ reasoning (str (or (:reasoning message) (:reasoning_content message)))
+ tool-calls (:tool_calls message)
+ truncated? (= "length" finish_reason)]
+ (cond
+ (str/includes? content "<think>")
+ (throw (preflight-ex
+ (tru "{0} streamed its reasoning as chat text. Restart vLLM with --reasoning-parser so thinking doesn''t appear inside Metabot''s answers."
+ (str model))))
+
+ (seq tool-calls)
+ (let [arguments (get-in (first tool-calls) [:function :arguments])]
+ (when-not (try
+ (map? (json/decode+kw (str arguments)))
+ (catch Exception _ false))
+ (throw (preflight-ex
+ (if truncated?
+ (tru "{0} reached the {1} token connection-test ceiling partway through a tool call. A model that generates this much before calling a tool is too slow to drive Metabot."
+ (str model) (str probe-max-tokens))
+ (tru "The vLLM server returned a tool call whose arguments are not valid JSON. The --tool-call-parser most likely does not match {0}''s output format."
+ (str model))))))
+ (not (str/blank? reasoning)))
+
+ (and truncated? (not (str/blank? reasoning)))
+ (throw (preflight-ex
+ (tru "{0} spent the entire {1} token connection-test budget reasoning without calling a tool. A model that thinks this long about a trivial prompt is too slow to drive Metabot."
+ (str model) (str probe-max-tokens))))
+
+ ;; A tool call cut off at the ceiling reaches here, not the `(seq tool-calls)` branch above: the
+ ;; parsers extract from complete output, so a call missing its closing sentinel yields no
+ ;; `tool_calls` at all and leaves the raw text in `content`. That is indistinguishable from
+ ;; prose except by `finish_reason`, and naming the flags — which are working, or the sentinel
+ ;; would not be there — sends the admin to fix something that is not broken. A verbose model
+ ;; whose flags really are missing lands here too, and gets the flags message on the retry after
+ ;; raising the ceiling; truncation is the problem to fix first either way.
+ truncated?
+ (throw (preflight-ex
+ (tru "{0} reached the {1} token connection-test ceiling before completing a tool call. A model that generates this much before calling a tool is too slow to drive Metabot."
+ (str model) (str probe-max-tokens))))
+
+ :else
+ (throw (preflight-ex
+ (tru "The vLLM server answered with text instead of calling a tool. Restart it with --enable-auto-tool-choice and a --tool-call-parser matching {0}''s output format."
+ (str model)))))))
+
+(defn- check-structured-output!
+ "Check that guided decoding works. A different failure from [[check-tool-calling!]]: a model whose
+ grammar the server cannot compile chats fine but breaks titling and the whole `sql` profile."
+ [auth model]
+ (let [{:keys [message finish_reason]} (probe-chat! auth model "required")]
+ (when (empty? (:tool_calls message))
+ (throw (preflight-ex
+ (if (= "length" finish_reason)
+ (tru "{0} reached the {1} token connection-test ceiling without producing a forced tool call. Metabot needs structured output support for conversation titles and SQL generation."
+ (str model) (str probe-max-tokens))
+ (tru "The vLLM server did not honor a forced tool call. Metabot needs structured output support for conversation titles and SQL generation.")))))))
+
+(defn- no-models-ex []
+ (preflight-ex (tru "The vLLM server is reachable but is not serving any models.")))
+
+(defn- connect-candidates
+ "Catalog entries eligible to be adopted when the connect path picks a model for the admin.
+
+ Skips LoRA adapters, whose `parent` names the base model they adapt. Nothing in the catalog
+ distinguishes an embedding or reranker deployment, so a multi-model server can still be adopted
+ from wrongly — the admin re-picks from the dropdown."
+ [entries]
+ (or (seq (remove :parent entries)) entries))
+
+(defn- probe-target
+ "The catalog entry [[preflight!]] will probe.
+
+ A requested model is the only acceptable target: falling back to another served model would pass
+ every check and then persist a provider string naming a model the server does not have. The
+ fallback is for the connect path, which supplies no model because the served name is knowable
+ only from this catalog."
+ [entries requested-model]
+ (if requested-model
+ (or (u/seek #(= requested-model (:id %)) entries)
+ (throw (if (seq entries)
+ (preflight-ex (tru "The vLLM server is not serving {0}. It is serving: {1}."
+ (str requested-model) (str/join ", " (map :id entries))))
+ (no-models-ex))))
+ (or (first (connect-candidates entries))
+ (throw (no-models-ex)))))
+
+(defn- await-probe!
+ "Deref a probe future, unwrapping the `ExecutionException` so the preflight's own `ex-info` reaches
+ the caller with its `:api-error` tag intact."
+ [fut]
+ (try
+ @fut
+ (catch ExecutionException e
+ (throw (or (ex-cause e) e)))))
+
+(defn- run-probes!
+ "Run both contract probes against `model` and return whether it streamed reasoning.
+
+ They run concurrently; deref order fixes the verdict, tool calling being the more actionable
+ diagnosis when a server fails both. The loser is cancelled rather than left generating against the
+ operator's server long after anyone is listening."
+ [auth model]
+ (try
+ (let [tool-calling (future (check-tool-calling! auth model))
+ structured (future (check-structured-output! auth model))]
+ (try
+ (let [reasoning? (await-probe! tool-calling)]
+ (await-probe! structured)
+ (boolean reasoning?))
+ (finally
+ (future-cancel tool-calling)
+ (future-cancel structured))))
+ (catch SocketTimeoutException _
+ (throw (preflight-ex
+ (tru "The vLLM server did not answer the connection test within {0}ms. Check that it is not overloaded — a server this slow to answer a trivial prompt cannot drive Metabot."
+ (str (:socket-timeout (probe-timeouts)))))))
+ (catch Exception e
+ (core/rethrow-api-error! "vllm" vllm-error-msg e))))
+
+(defn- preflight!
+ "Exercise the contract the agent loop depends on, against the model that will actually be used, and
+ return `{:model id :reasoning? bool}`. The connect path must adopt exactly this model rather than
+ re-deriving it from the listing, which agrees only while nothing reorders the catalog.
+
+ `:reasoning?` reports whether the probed model streamed reasoning. Only the probe can answer that,
+ and the answer drives which renderer the frontend picks, so the connection records it (see
+ [[reasoning-config-key]])."
+ [auth entries requested-model]
+ (let [entry (probe-target entries requested-model)
+ model (:id entry)]
+ (check-context-budget! entry)
+ {:model model
+ :reasoning? (run-probes! auth model)}))
+
+(defn list-models
+ "List the models the connection's vLLM server is serving. Pass-through: there is nothing to
+ whitelist, and `display_name` falls back to the served id.
+
+ `:probe?` additionally runs [[preflight!]], and reports the model it exercised as `:probed-model`
+ for the connect path to adopt, along with the `:learned-config` the probe determined. Reserved for
+ the connect and edit paths — a tool-call probe on every model listing would stall the admin picker
+ behind a full prefill."
+ ([] (list-models {}))
+ ([{:keys [credentials ai-proxy? model probe?]}]
+ (let [auth (vllm-auth credentials ai-proxy?)
+ entries (list-all-models auth)
+ probed (when probe?
+ (preflight! auth entries model))]
+ (cond-> {:models (mapv (fn [{:keys [id] :as entry}]
+ {:id id :display_name (or (:name entry) id)})
+ entries)}
+ probed (assoc :probed-model (:model probed)
+ :learned-config {reasoning-config-key (str (:reasoning? probed))})))))
+
+;;; --------------------------------------------------- Requests -------------------------------------------------
+
+(mu/defn vllm-request-body
+ "Build the Chat Completions request body for an LLM request.
+
+ Matches what [[chat-completions/request-body]] emits, except that `max_tokens` is always sent —
+ without a ceiling vLLM falls back to the remaining context window, so one looping small model
+ consumes the whole budget in a single call — and is raised to
+ [[forced-tool-call-token-floor]] or [[reasoning-model-token-floor]] where either applies, and
+ `temperature` falls back to [[default-temperature]]. All three stay adapter-local rather than
+ moving into the shared builder, which would also change Z.AI, Mistral, and OpenRouter."
+ [{:keys [max-tokens temperature schema tool_choice credentials] :as opts} :- core/LLMRequestOpts]
+ (let [forced? (or (some? schema) (= "required" (some-> tool_choice name)))]
+ (assoc (chat-completions/request-body (cond-> opts
+ (nil? temperature) (assoc :temperature default-temperature)))
+ :max_tokens (cond-> (or max-tokens (llm/llm-max-tokens))
+ forced? (max forced-tool-call-token-floor)
+ (reasoning-connection? credentials) (max reasoning-model-token-floor)))))
+
+(defn- stream-io-ex
+ "The vLLM error for a transport failure while *consuming* a response stream. Tagged
+ `:retryable? false`: on a self-hosted server a stalled or severed response means \"too slow\" or
+ \"it died\", not \"transient\", and a retry replays a full cold prefill at up to
+ `llm-vllm-request-timeout-ms` (300s) apiece. The tag is required — `retryable-error?` walks the
+ cause chain and would otherwise match the `IOException` below."
+ [^IOException e timeout-ms]
+ (if (instance? SocketTimeoutException e)
+ (ex-info (tru "The vLLM server stopped responding after {0}ms. Raise the vLLM request timeout, or serve a faster model."
+ (str timeout-ms))
+ {:api-error true
+ :error-code :vllm-timeout
+ :retryable? false}
+ e)
+ (ex-info (tru "The connection to the vLLM server was interrupted before the response finished.")
+ {:api-error true
+ :error-code :vllm-stream-interrupted
+ :retryable? false}
+ e)))
+
+(defn- request-io-ex
+ "The vLLM error for a transport failure while *establishing* a request. `core/rethrow-api-error!`
+ would render these as \"vllm API request failed: Read timed out\", naming neither the server's
+ slowness nor the setting that governs it.
+
+ Tagged `:retryable? false` for the same reason as [[stream-io-ex]], and more importantly: nothing
+ has been emitted yet, so `call-llm`'s own \"nothing emitted\" predicate would not stop a replay."
+ [^IOException e base-url timeout-ms]
+ (if (instance? SocketTimeoutException e)
+ (ex-info (tru "The vLLM server did not respond within {0}ms. Check that it is not overloaded, or raise the vLLM request timeout."
+ (str timeout-ms))
+ {:api-error true
+ :error-code :vllm-timeout
+ :retryable? false}
+ e)
+ (unreachable-ex e base-url {:retryable? false})))
+
+(defn- io-guarded
+ "Wrap a stream reducible so a transport failure while consuming it surfaces as [[stream-io-ex]]
+ rather than a raw `IOException`. The adapter's own `try` covers only establishing the request.
+
+ Goes inside `core/reducible-with-api-errors`, never outside: [[stream-io-ex]] tags `:api-error
+ true`, which `core/rethrow-api-error!` rethrows unchanged, so this translation wins for IO."
+ [reducible timeout-ms]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ rf init]
+ (try
+ (.reduce ^clojure.lang.IReduceInit reducible rf init)
+ (catch IOException e
+ (throw (stream-io-ex e timeout-ms)))))))
+
+(mu/defn vllm-raw
+ "Perform a streaming request to a vLLM server's Chat Completions API.
+ Opts map takes `:credentials` (`{:base-url ... :api-key ...}`) from the connection serving this
+ request, and throws without a base URL.
+ `:ai-proxy?` is not supported for vLLM and throws when true."
+ [{:keys [model tools credentials ai-proxy?] :as opts} :- core/LLMRequestOpts]
+ (when ai-proxy?
+ (throw (ai-proxy-unsupported-ex)))
+ (when (str/blank? model)
+ (throw (missing-model-ex)))
+ (let [req (vllm-request-body opts)
+ timeout-ms (llm/llm-vllm-request-timeout-ms)]
+ (log/debug "vLLM request" {:model model :msg-count (count (:messages req)) :tools (count (or tools []))})
+ (with-span :info {:name :metabot.vllm/request
+ :model model
+ :msg-count (count (:messages req))
+ :tool-count (count (or tools []))}
+ (try
+ (let [auth (vllm-auth credentials ai-proxy?)
+ response (core/request auth
+ (merge {:method :post
+ :url "/chat/completions"
+ :as :stream
+ :headers {"Content-Type" "application/json"}
+ :body (json/encode req)}
+ (inference-timeouts)))]
+ (-> (core/sse-reducible (:body response))
+ (debug/capture-stream {:provider "vllm"
+ :model model
+ :url "/chat/completions"
+ :request req})
+ (io-guarded timeout-ms)
+ (core/reducible-with-api-errors "vllm" vllm-error-msg)))
+ ;; Ordered: clj-http raises an `IOException` only when there is no response at all, so this
+ ;; cannot swallow one `vllm-error-msg` would have translated.
+ (catch IOException e
+ (throw (request-io-ex e (:base-url credentials) timeout-ms)))
+ (catch Exception e
+ (core/rethrow-api-error! "vllm" vllm-error-msg e))))))
+
+(defn vllm->aisdk-chunks-xf
+ "Translates vLLM Chat Completions streaming chunks into AI SDK v5 protocol chunks.
+
+ A model started with `--reasoning-parser` routes thinking to `delta.reasoning` and its answer back
+ to `delta.content`; both are forwarded. The branch is self-gating — the field is present only when
+ the served model reasons — so nothing here needs to know which model is loaded.
+
+ vLLM adds no `finish_reason` beyond OpenAI's, so it takes the base stop-reason table."
+ []
+ (chat-completions/chat-completions->aisdk-chunks-xf chat-completions/stop-reasons
+ {:forward-reasoning? true}))
+
+(defn vllm
+ "Call a vLLM server's Chat Completions API, return AISDK stream."
+ [& args]
+ (let [raw (apply vllm-raw args)]
+ (eduction (vllm->aisdk-chunks-xf) raw)))
diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.clj
index 94483b13ffc..d81cf99c861 100644
--- a/src/metabase/metabot/settings.clj
+++ b/src/metabase/metabot/settings.clj
@@ -5,6 +5,7 @@
[metabase.llm.settings :as llm.settings]
[metabase.metabot.self.claude :as claude]
[metabase.metabot.self.openai :as openai]
+ [metabase.metabot.self.vllm :as vllm]
[metabase.settings.core :as setting :refer [defsetting]]
[metabase.util.i18n :refer [deferred-tru tru]]
[metabase.util.log :as log]))
@@ -221,12 +222,17 @@
:doc false)
(defn- llm-provider-streams-reasoning?
- "Whether a model reference names a model that streams its reasoning back to us."
+ "Whether a model reference names a model that streams its reasoning back to us.
+
+ Anthropic and OpenAI answer from the model name, because thinking is requested in the request body. vLLM answers
+ from what its connect-time probe observed and recorded on the connection — the flag depends on the operator's
+ `--reasoning-parser` as well as on the model, so the name cannot settle it."
[model-ref]
- (let [{:keys [type model]} (llm.provider/resolve-model-ref model-ref)]
+ (let [{:keys [type model credentials]} (llm.provider/resolve-model-ref model-ref)]
(case type
"anthropic" (claude/reasoning-model? model)
"openai" (openai/reasoning-model? model)
+ "vllm" (vllm/reasoning-connection? credentials)
false)))
(defsetting llm-metabot-supports-reasoning?
diff --git a/test/metabase/llm/api/provider_test.clj b/test/metabase/llm/api/provider_test.clj
index 5463dd65a84..4ef2509c90b 100644
--- a/test/metabase/llm/api/provider_test.clj
+++ b/test/metabase/llm/api/provider_test.clj
@@ -1,5 +1,6 @@
(ns metabase.llm.api.provider-test
(:require
+ [clj-http.client :as http]
[clojure.test :refer [deftest is testing use-fixtures]]
[metabase.llm.api.provider :as llm.api.provider]
[metabase.llm.provider :as llm.provider]
@@ -30,9 +31,10 @@
(deftest provider-types-test
(testing "every provider type is listed with the credential fields a connection needs"
(let [types (mt/user-http-request :crowberto :get 200 "llm/provider-types")]
- (is (= #{"anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "metabase"}
+ (is (= #{"anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "vllm"
+ "metabase"}
(set (map :type types))))
- (is (= ["anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock"]
+ (is (= ["anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "vllm"]
(remove #{"metabase"} (map :type types)))
"the bring-your-own-key providers keep their registry order")
(is (=? {:type "anthropic"
@@ -234,6 +236,97 @@
{:type "openai" :config {:api-key "sk-valid"}})
(is (= "anthropic/claude-opus-4-8" (metabot.settings/llm-metabot-provider))))))))
+(deftest create-vllm-connection-adopts-the-model-its-probe-exercised-test
+ (testing (str "A vLLM server serves whatever the operator loaded, so there is no default model to select: "
+ "connecting adopts the model the connect-time probe actually ran the agent-loop contract "
+ "against, and records what that probe could only learn by running.")
+ (let [opts (atom nil)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [_provider o]
+ (reset! opts o)
+ {:models [{:id "vllm-test" :display_name "vllm-test"}
+ {:id "other" :display_name "other"}]
+ :probed-model "vllm-test"
+ :learned-config {:model-reasoning "true"}})]
+ (mt/with-temporary-setting-values [llm-providers []]
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider nil]
+ (is (=? {:key "vllm"
+ :type "vllm"
+ :usable true
+ :config {:base-url "http://vllm.internal:8000/v1"}}
+ (mt/user-http-request :crowberto :post 200 "llm/providers"
+ {:type "vllm"
+ :config {:base-url "http://vllm.internal:8000/v1"}})))
+ (testing "the connection is verified by a probe, not by a plain listing"
+ (is (=? {:credentials {:base-url "http://vllm.internal:8000/v1"} :probe? true} @opts)))
+ (is (= "vllm/vllm-test" (metabot.settings/llm-metabot-provider)))
+ (testing "and what the probe learned is stored on the connection, where the request path reads it"
+ (is (= {:base-url "http://vllm.internal:8000/v1"
+ :model-reasoning "true"}
+ (stored-config "vllm")))
+ (is (true? (metabot.settings/llm-metabot-supports-reasoning?))))))))))
+
+(deftest create-vllm-connection-is-rejected-when-the-probe-fails-test
+ (testing "a server that cannot drive the agent loop is not saved, so nothing points Metabot at it"
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [& _]
+ (throw (ex-info "The vLLM server answered with text instead of calling a tool."
+ {:api-error true :status-code 400})))]
+ (mt/with-temporary-setting-values [llm-providers []]
+ (is (=? {:message "The vLLM server answered with text instead of calling a tool."}
+ (mt/user-http-request :crowberto :post 400 "llm/providers"
+ {:type "vllm"
+ :config {:base-url "http://vllm.internal:8000/v1"}})))
+ (is (= [] (llm.provider/connections)))))))
+
+(deftest create-vllm-connection-rejects-an-unreachable-base-url-test
+ (testing (str "vLLM is the one provider whose base URL the admin types, so a typo has to come back as the "
+ "adapter's message on the form — not as the 500 an untagged transport failure would produce.")
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (java.net.ConnectException. "Connection refused")))]
+ (mt/with-temporary-setting-values [llm-providers []]
+ (is (= (str "Could not reach the vLLM server at http://vllm.internal:8000/v1. "
+ "Check that it is running and that the base URL is correct.")
+ (:message (mt/user-http-request :crowberto :post 400 "llm/providers"
+ {:type "vllm"
+ :config {:base-url "http://vllm.internal:8000/v1"}}))))
+ (is (= [] (llm.provider/connections)))))))
+
+(deftest models-listing-does-not-probe-test
+ (testing "listing models is a page load; only a write may spend a generation on the operator's server"
+ (let [opts (atom nil)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [_provider o]
+ (reset! opts o)
+ {:models [{:id "vllm-test" :display_name "vllm-test"}]})]
+ (mt/with-temporary-setting-values [llm-providers [(connection "vllm" "vllm"
+ {:base-url "http://vllm.internal:8000/v1"})]]
+ (is (=? [{:key "vllm" :models [{:id "vllm-test"}]}]
+ (mt/user-http-request :crowberto :get 200 "llm/models")))
+ (is (not (:probe? @opts))))))))
+
+(deftest update-verifies-against-the-model-the-connection-serves-test
+ (testing (str "An edit that names no model is verified against the model this connection is actually serving "
+ "Metabot, rather than whichever one the provider happens to list first.")
+ (let [opts (atom nil)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [_provider o]
+ (reset! opts o)
+ {:models [{:id "served-a" :display_name "served-a"}]
+ :probed-model "served-b"
+ :learned-config {:model-reasoning "false"}})]
+ (mt/with-temporary-setting-values [llm-providers [(connection "vllm" "vllm"
+ {:base-url "http://old.internal:8000/v1"
+ :model-reasoning "true"})]]
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider "vllm/served-b"]
+ (mt/user-http-request :crowberto :put 200 "llm/providers/vllm"
+ {:config {:base-url "http://vllm.internal:8000/v1"}})
+ (is (=? {:model "served-b" :probe? true} @opts))
+ (testing "and the probe's fresh verdict replaces the one the connection was carrying"
+ (is (= {:base-url "http://vllm.internal:8000/v1"
+ :model-reasoning "false"}
+ (stored-config "vllm")))
+ (is (false? (metabot.settings/llm-metabot-supports-reasoning?))))))))))
+
(deftest create-selects-a-model-composed-from-the-config-test
(testing (str "Azure names its deployment in `:config` rather than listing models, and the form leaves a field it "
"pre-filled with the registry default out of the payload, so the default has to be filled in before "
diff --git a/test/metabase/llm/provider_test.clj b/test/metabase/llm/provider_test.clj
index 4e42cd7654a..0aaf60f8221 100644
--- a/test/metabase/llm/provider_test.clj
+++ b/test/metabase/llm/provider_test.clj
@@ -218,6 +218,13 @@
"connection without one still counts as complete")
(is (true? (llm.provider/config-complete? "azure" {:api-key "azure-key"
:base-url "https://r.services.ai.azure.com/openai"})))))
+ (testing "vLLM needs a base URL, and its API key is optional — a server started without --api-key takes none"
+ (is (true? (llm.provider/config-complete? "vllm" {:base-url "http://vllm.internal:8000/v1"})))
+ (is (true? (llm.provider/config-complete? "vllm" {:base-url "http://vllm.internal:8000/v1"
+ :api-key "local-dev-key"})))
+ (is (false? (llm.provider/config-complete? "vllm" {:api-key "local-dev-key"})))
+ (is (false? (llm.provider/config-complete? "vllm" {:base-url " "})))
+ (is (false? (llm.provider/config-complete? "vllm" nil))))
(testing "bedrock needs both AWS keys, and neither the region nor the session token"
(is (true? (llm.provider/config-complete? "bedrock" {:access-key-id "AKIAIOSFODNN7EXAMPLE"
:secret-access-key "test-secret"})))
@@ -256,6 +263,17 @@
:config {:api-key "sk-ant-db"}
:source :db}]
(llm.provider/connections)))))
+ (testing "stored and env-derived connections are merged, stored first"
+ (mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic" {:api-key "sk-ant-db"})]]
+ (mt/with-temp-env-var-value! [mb-llm-openai-api-key "sk-env"]
+ (is (= [["anthropic" :db] ["openai" :env]]
+ (map (juxt :key :source) (llm.provider/connections)))))))
+ (testing "the whole stored list is read-only when it comes from the environment"
+ (mt/with-temp-env-var-value! [mb-llm-providers "[{\"key\":\"anthropic\",\"type\":\"anthropic\",\"name\":\"Anthropic\",\"config\":{\"api-key\":\"sk-ant-env\"}}]"]
+ (is (= [["anthropic" :env]]
+ (map (juxt :key :source) (llm.provider/connections)))))))
+
+(deftest connections-synthesized-from-the-environment-test
(testing "the single-provider environment variables synthesize read-only connections keyed by provider type"
(mt/with-temporary-setting-values [llm-providers []]
(mt/with-temp-env-var-value! [mb-llm-anthropic-api-key "sk-ant-env"
@@ -272,11 +290,29 @@
(mt/with-temporary-setting-values [llm-providers []]
(mt/with-temp-env-var-value! [mb-llm-anthropic-api-base-url "https://env.example.com"]
(is (= [] (llm.provider/connections))))))
- (testing "stored and env-derived connections are merged, stored first"
- (mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic" {:api-key "sk-ant-db"})]]
- (mt/with-temp-env-var-value! [mb-llm-openai-api-key "sk-env"]
- (is (= [["anthropic" :db] ["openai" :env]]
- (map (juxt :key :source) (llm.provider/connections)))))))
+ (testing "vLLM's base URL is its credential, so it alone synthesizes a connection"
+ (mt/with-temporary-setting-values [llm-providers []]
+ (mt/with-temp-env-var-value! [mb-llm-vllm-api-base-url "http://vllm.internal:8000/v1"]
+ (is (= [{:key "vllm"
+ :type "vllm"
+ :name "vLLM"
+ :source :env
+ :env-vars #{"MB_LLM_VLLM_API_BASE_URL"}
+ :env-fields #{:base-url}
+ :config {:base-url "http://vllm.internal:8000/v1"}}]
+ (llm.provider/connections))))
+ (testing "and a key on its own does not: it authenticates nothing without a server to send it to"
+ (mt/with-temp-env-var-value! [mb-llm-vllm-api-key "local-dev-key"]
+ (is (= [] (llm.provider/connections)))))))
+ (testing "a metabase/ reference pinned by the environment synthesizes the managed connection it names"
+ (mt/with-temporary-setting-values [llm-providers []]
+ (mt/with-temp-env-var-value! [mb-llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"]
+ (is (=? [{:key "metabase" :type "metabase" :source :env}]
+ (llm.provider/connections)))
+ (is (=? {:connection-key "metabase" :type "anthropic" :ai-proxy? true}
+ (llm.provider/resolve-model-ref "metabase/anthropic/claude-sonnet-4-6")))))))
+
+(deftest connections-shadowed-by-the-environment-test
(testing "the environment shadows a stored connection with the same key field by field, not wholesale"
(mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic"
{:api-key "sk-ant-db"
@@ -296,18 +332,7 @@
(mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic" {:api-key "sk-ant-db"})]]
(mt/with-temp-env-var-value! [mb-llm-anthropic-api-base-url "https://env.example.com"]
(is (= {:api-key "sk-ant-db" :base-url "https://env.example.com"}
- (llm.provider/credentials "anthropic"))))))
- (testing "a metabase/ reference pinned by the environment synthesizes the managed connection it names"
- (mt/with-temporary-setting-values [llm-providers []]
- (mt/with-temp-env-var-value! [mb-llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"]
- (is (=? [{:key "metabase" :type "metabase" :source :env}]
- (llm.provider/connections)))
- (is (=? {:connection-key "metabase" :type "anthropic" :ai-proxy? true}
- (llm.provider/resolve-model-ref "metabase/anthropic/claude-sonnet-4-6"))))))
- (testing "the whole stored list is read-only when it comes from the environment"
- (mt/with-temp-env-var-value! [mb-llm-providers "[{\"key\":\"anthropic\",\"type\":\"anthropic\",\"name\":\"Anthropic\",\"config\":{\"api-key\":\"sk-ant-env\"}}]"]
- (is (= [["anthropic" :env]]
- (map (juxt :key :source) (llm.provider/connections)))))))
+ (llm.provider/credentials "anthropic")))))))
(deftest stored-connections-keeps-a-connection-the-environment-shadows-test
(testing (str "The stored list keeps the credentials the environment shadows, so writes rebuild from here and "
@@ -433,7 +458,8 @@
"so a type without a decided logo fails to compile. Nothing links the two, so adding a type here "
"without updating them ships a provider that silently falls back to the generic icon. Update "
"both, then this list.")
- (is (= #{"anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "metabase"}
+ (is (= #{"anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "vllm"
+ "metabase"}
(into #{} (map :type) (llm.provider/provider-types))))))
(deftest ^:parallel provider-types-test
@@ -450,6 +476,7 @@
(is (= #{:api-key} (llm.provider/secret-field-keys "anthropic")))
(is (= #{:api-key} (llm.provider/secret-field-keys "azure")))
(is (= #{:access-key-id :secret-access-key :session-token} (llm.provider/secret-field-keys "bedrock")))
+ (is (= #{:api-key} (llm.provider/secret-field-keys "vllm")))
(testing "google's service account key is a file field, but it is the whole credential"
(is (= #{:service-account-key :oauth-access-token} (llm.provider/secret-field-keys "google"))))
(is (= #{} (llm.provider/secret-field-keys "metabase"))))
@@ -464,6 +491,9 @@
;; azure's models are deployment names the admin chooses, so there is nothing to default to
"azure" nil
"bedrock" "anthropic.claude-opus-4-8"
+ ;; nor is there for vLLM, which serves whatever the operator loaded: connecting adopts the model
+ ;; its probe exercised
+ "vllm" nil
"metabase" "anthropic/claude-sonnet-4-6"}
(into {} (map (juxt :type #(llm.provider/default-model (:type %)))) (llm.provider/provider-types))))
(is (nil? (llm.provider/default-model "evilai"))))
@@ -479,6 +509,8 @@
"google" nil
"azure" nil
"bedrock" "anthropic.claude-haiku-4-5"
+ ;; a vLLM server serves the one model the operator loaded, so there is no cheaper tier to fall back to
+ "vllm" nil
"metabase" nil}
(into {} (map (juxt :type #(llm.provider/mini-model (:type %)))) (llm.provider/provider-types))))
(is (nil? (llm.provider/mini-model "evilai"))))
diff --git a/test/metabase/llm/test_util.clj b/test/metabase/llm/test_util.clj
index 09be007ee17..6f07e268447 100644
--- a/test/metabase/llm/test_util.clj
+++ b/test/metabase/llm/test_util.clj
@@ -23,6 +23,7 @@
"bedrock" {:access-key-id "AKIAIOSFODNN7EXAMPLE"
:secret-access-key "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
:region "us-east-1"}
+ "vllm" {:base-url "http://vllm.internal:8000/v1"}
"metabase" {}})
(defn connection
diff --git a/test/metabase/metabot/self/openai/chat_completions_test.clj b/test/metabase/metabot/self/openai/chat_completions_test.clj
index 0464f1df81e..ae5d6cc0721 100644
--- a/test/metabase/metabot/self/openai/chat_completions_test.clj
+++ b/test/metabase/metabot/self/openai/chat_completions_test.clj
@@ -308,6 +308,19 @@
#"Mistral returned an unexpected model list response"
(chat-completions/models-catalog "Mistral" {:status 200 :body body})))))))
+(deftest ^:parallel models-catalog-appends-a-caller-supplied-detail-test
+ (testing "a supplied detail is appended to the message"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"vLLM returned an unexpected model list response\. Check http://vllm\.internal:8000/v1\."
+ (chat-completions/models-catalog "vLLM" {:status 200 :body {:object "list"}}
+ {:detail "Check http://vllm.internal:8000/v1."}))))
+ (testing "and a caller that supplies none gets the bare message"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Mistral returned an unexpected model list response$"
+ (chat-completions/models-catalog "Mistral" {:status 200 :body {:object "list"}} nil)))))
+
(deftest ^:parallel models-catalog-error-is-tagged-api-error-without-status-test
(testing "the thrown error is tagged :api-error so rethrow-api-error! passes it through, and carries no status"
;; `metabase.metabot.api/provider-client-error?` renders any 4xx :api-error under the admin API-key
diff --git a/test/metabase/metabot/self/openrouter_test.clj b/test/metabase/metabot/self/openrouter_test.clj
index 81a1eac711a..c3414f9f935 100644
--- a/test/metabase/metabot/self/openrouter_test.clj
+++ b/test/metabase/metabot/self/openrouter_test.clj
@@ -72,6 +72,44 @@
:input [{:role :user :content "hi"}]})]
(is (= ["user"] (map :role (:messages body)))))))
+;;; ──────────────────────────────────────────────────────────────────
+;;; Temperature gating
+;;; ──────────────────────────────────────────────────────────────────
+
+(defn- request-body-temperature
+ [model]
+ (:temperature (openrouter/openrouter-request-body {:model model
+ :input [{:role :user :content "hi"}]
+ :temperature 0.3})))
+
+(deftest ^:parallel request-body-drops-temperature-for-models-that-reject-it-test
+ (testing "the GPT-5 and o-series families take no explicit temperature"
+ (doseq [model ["openai/gpt-5.6-sol" "openai/gpt-5.5-pro" "openai/gpt-5.4" "openai/gpt-5.4-mini"
+ "openai/o1" "openai/o3-mini"]]
+ (testing model
+ (is (nil? (request-body-temperature model))))))
+ (testing "neither does current-generation Claude, whose OpenRouter ids use dots for the minor version"
+ (doseq [model ["anthropic/claude-fable-5" "anthropic/claude-opus-5" "anthropic/claude-opus-4.8"
+ "anthropic/claude-opus-4.7" "anthropic/claude-sonnet-5"]]
+ (testing model
+ (is (nil? (request-body-temperature model)))))))
+
+(deftest ^:parallel request-body-keeps-temperature-for-models-that-accept-it-test
+ (testing "every other whitelisted model still gets the profile's temperature"
+ (doseq [model ["anthropic/claude-opus-4.6" "anthropic/claude-opus-4.5" "anthropic/claude-opus-4.1"
+ "anthropic/claude-sonnet-4.6" "anthropic/claude-sonnet-4.5" "anthropic/claude-haiku-4.5"
+ "deepseek/deepseek-v4-pro" "mistralai/mistral-medium-3-5" "z-ai/glm-5.2"]]
+ (testing model
+ (is (= 0.3 (request-body-temperature model)))))))
+
+(deftest ^:parallel request-body-omits-temperature-when-none-is-supplied-test
+ (testing "a request with no temperature is unchanged either way"
+ (doseq [model ["openai/gpt-5.4" "anthropic/claude-haiku-4.5"]]
+ (testing model
+ (is (not (contains? (openrouter/openrouter-request-body {:model model
+ :input [{:role :user :content "hi"}]})
+ :temperature)))))))
+
;;; ──────────────────────────────────────────────────────────────────
;;; openrouter-request-body tool_choice tests
;;; ──────────────────────────────────────────────────────────────────
diff --git a/test/metabase/metabot/self/vllm_test.clj b/test/metabase/metabot/self/vllm_test.clj
new file mode 100644
index 00000000000..43b99d27f8b
--- /dev/null
+++ b/test/metabase/metabot/self/vllm_test.clj
@@ -0,0 +1,1098 @@
+(ns metabase.metabot.self.vllm-test
+ (:require
+ [clj-http.client :as http]
+ [clojure.test :refer :all]
+ [metabase.llm.settings :as llm.settings]
+ [metabase.metabot.self :as self]
+ [metabase.metabot.self.core :as self.core]
+ [metabase.metabot.self.debug :as debug]
+ [metabase.metabot.self.openai.chat-completions :as chat-completions]
+ [metabase.metabot.self.vllm :as vllm]
+ [metabase.metabot.test-util :as metabot.tu]
+ [metabase.test :as mt]
+ [metabase.util.json :as json])
+ (:import
+ (java.net ConnectException SocketTimeoutException)
+ (java.util.concurrent CountDownLatch TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private base-url "http://vllm.internal:8000/v1")
+
+(def ^:private credentials
+ "What a resolved vLLM connection hands the adapter: adapters read credentials only, never settings."
+ {:base-url base-url})
+
+(def ^:private keyed-credentials
+ "A connection to a server started with --api-key."
+ (assoc credentials :api-key "local-dev-key"))
+
+(def ^:private reasoning-credentials
+ "A connection whose connect-time probe found the served model streaming its reasoning."
+ (assoc credentials vllm/reasoning-config-key "true"))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; vllm-request-body
+;;; ──────────────────────────────────────────────────────────────────
+
+(deftest ^:parallel request-body-applies-default-max-tokens-test
+ (testing "an explicit max_tokens is always sent — without one vLLM falls back to the whole remaining context window"
+ (is (= (llm.settings/llm-max-tokens)
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]}))))))
+
+(deftest ^:parallel request-body-caller-max-tokens-wins-test
+ (testing "a caller-supplied max-tokens is not overridden by the default"
+ (is (= 128
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :max-tokens 128}))))))
+
+(deftest ^:parallel request-body-raises-max-tokens-for-a-forced-tool-call-test
+ (testing "a forced tool call gets the token floor even when the caller asked for less"
+ (testing "schema"
+ (is (= 2048
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :schema {:type "object"}
+ :max-tokens 128})))))
+ (testing "tool_choice required"
+ (is (= 2048
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :tools [(metabot.tu/get-time-tool)]
+ :tool_choice "required"
+ :max-tokens 128})))))))
+
+(deftest ^:parallel request-body-floor-never-lowers-a-ceiling-test
+ (testing "a caller-supplied ceiling above the floor is left alone"
+ (is (= 8192
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :schema {:type "object"}
+ :max-tokens 8192}))))))
+
+(deftest ^:parallel request-body-floor-applies-only-to-forced-tool-calls-test
+ (testing "an unforced request keeps the caller's ceiling — the floor exists for thinking that precedes a tool call"
+ (is (= 128
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :tools [(metabot.tu/get-time-tool)]
+ :tool_choice "auto"
+ :max-tokens 128}))))))
+
+(deftest ^:parallel request-body-raises-max-tokens-for-a-reasoning-model-test
+ (testing "the agent path forces nothing and supplies no ceiling, so a reasoning model would otherwise get
+ the shared 4096 default for thinking, answer, and tool call combined"
+ (is (= 16384
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials reasoning-credentials}))))
+ (testing "and a model the probe found does not reason keeps the default"
+ (is (= (llm.settings/llm-max-tokens)
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials credentials})))))))
+
+(deftest ^:parallel request-body-reasoning-floor-outranks-the-forced-tool-call-floor-test
+ (testing "a reasoning model has to clear its thinking before the forced call, so the higher floor wins"
+ (is (= 16384
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :schema {:type "object"}
+ :max-tokens 128
+ :credentials reasoning-credentials}))))))
+
+(deftest ^:parallel request-body-reasoning-floor-never-lowers-a-ceiling-test
+ (testing "a caller asking for more than the floor is left alone"
+ (is (= 32000
+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :max-tokens 32000
+ :credentials reasoning-credentials}))))))
+
+(deftest ^:parallel request-body-no-default-model-test
+ (testing "the model is never defaulted — a vLLM server's model name is whatever the operator loaded"
+ (is (nil? (:model (vllm/vllm-request-body {:input [{:role :user :content "hi"}]}))))))
+
+(deftest ^:parallel request-body-system-and-stream-test
+ (testing "the system prompt leads the messages and the request streams with usage included"
+ (let [body (vllm/vllm-request-body {:model "vllm-test"
+ :system "You are a helpful assistant."
+ :input [{:role :user :content "hi"}]})]
+ (is (= {:role "system" :content "You are a helpful assistant."}
+ (-> body :messages first)))
+ (is (true? (:stream body)))
+ (is (= {:include_usage true} (:stream_options body))))))
+
+(deftest ^:parallel request-body-tools-test
+ (testing "tools are sent in OpenAI function format with tool_choice auto"
+ (is (=? {:tools [{:type "function"
+ :function {:name "get-time"}}]
+ :tool_choice "auto"}
+ (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :tools [(metabot.tu/get-time-tool)]})))))
+
+(deftest ^:parallel request-body-schema-forces-structured-output-test
+ (testing "a schema forces a structured_output tool call"
+ (is (=? {:tools [{:type "function"
+ :function {:name "structured_output"
+ :parameters {:type "object"}}}]
+ :tool_choice "required"}
+ (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :schema {:type "object"
+ :properties {:title {:type "string"}}}})))))
+
+(deftest ^:parallel request-body-temperature-test
+ (testing "a caller's temperature passes through"
+ (is (= 0.7
+ (:temperature (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :temperature 0.7})))))
+ (testing "temperature 0 is honoured rather than treated as absent"
+ (is (= 0
+ (:temperature (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :temperature 0}))))))
+
+(deftest ^:parallel request-body-supplies-a-default-temperature-test
+ (testing "a caller that supplies none gets the adapter default rather than vLLM's own 1.0"
+ (is (= @#'vllm/default-temperature
+ (:temperature (vllm/vllm-request-body {:model "vllm-test"
+ :input [{:role :user :content "hi"}]}))))))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; Streaming chunk conversion
+;;;
+;;; Recorded verbatim from vLLM 0.26.0 serving Qwen3-14B with
+;;; `--tool-call-parser hermes`.
+;;; ──────────────────────────────────────────────────────────────────
+
+(def ^:private recorded-tool-call-stream
+ (let [msg-id "chatcmpl-ba409270f617984a"
+ arg (fn [s] {:id msg-id :model "vllm-test"
+ :choices [{:index 0 :delta {:content nil :tool_calls [{:index 0 :function {:arguments s}}]}
+ :finish_reason nil}]})]
+ (into [{:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:role "assistant" :content ""} :finish_reason nil}]}
+ {:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:content "<think>"} :finish_reason nil}]}
+ {:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:content "\n\n"} :finish_reason nil}]}
+ {:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:content "</think>"} :finish_reason nil}]}
+ {:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:content "\n\n"} :finish_reason nil}]}
+ {:id msg-id :model "vllm-test"
+ :choices [{:index 0
+ :delta {:content nil
+ :tool_calls [{:id "chatcmpl-tool-acf1cddb0052df26"
+ :type "function"
+ :index 0
+ :function {:name "record_table_name"}}]}
+ :finish_reason nil}]}]
+ cat
+ [(map arg ["{\"" "table" "_name" "\":" " \"" "orders" "\"}"])
+ [{:id msg-id :model "vllm-test" :choices [{:index 0 :delta {} :finish_reason "tool_calls"}]}
+ {:id msg-id :model "vllm-test" :choices []
+ :usage {:prompt_tokens 163 :total_tokens 189 :completion_tokens 26}}]])))
+
+(deftest ^:parallel recorded-streamed-tool-call-conv-test
+ (testing "a tool call whose id arrives only on the first delta accumulates into one tool-input"
+ (let [parts (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)) recorded-tool-call-stream)]
+ (is (=? {:type :tool-input
+ :id "chatcmpl-tool-acf1cddb0052df26"
+ :function "record_table_name"
+ :arguments {:table_name "orders"}}
+ (first (filter #(= :tool-input (:type %)) parts)))))))
+
+(deftest ^:parallel recorded-usage-has-no-cache-details-test
+ (testing "vLLM reports no prompt_tokens_details, so the cache columns come through as zero rather than throwing"
+ (let [parts (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)) recorded-tool-call-stream)]
+ (is (=? {:type :usage
+ :model "vllm-test"
+ :usage {:promptTokens 163
+ :completionTokens 26
+ :cacheCreationTokens 0
+ :cacheReadTokens 0}}
+ (first (filter #(= :usage (:type %)) parts)))))))
+
+(deftest ^:parallel think-tags-reach-the-text-stream-test
+ (testing "without --reasoning-parser a model's thinking arrives in delta.content and streams as assistant text"
+ (let [text (->> (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)) recorded-tool-call-stream)
+ (filter #(= :text (:type %)))
+ (map :text)
+ (apply str))]
+ (is (re-find #"<think>" text)))))
+
+;;; Recorded from vLLM 0.26.0 serving Qwen3-14B with `--reasoning-parser qwen3`. The shape that
+;;; could not be guessed: the field is `reasoning`, not `reasoning_content` — 0.26 renamed it and
+;;; documents the old spelling as deprecated. The opening chunk carries `content ""`, which must not
+;;; open a text block ahead of the reasoning.
+(def ^:private reasoning-stream
+ [{:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"
+ :choices [{:index 0 :delta {:role "assistant" :content ""} :finish_reason nil}]}
+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"
+ :choices [{:index 0 :delta {:reasoning "\nOkay"} :finish_reason nil}]}
+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"
+ :choices [{:index 0 :delta {:reasoning ", the user asks 2+2."} :finish_reason nil}]}
+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"
+ :choices [{:index 0 :delta {:content "Four"} :finish_reason nil}]}
+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"
+ :choices [{:index 0 :delta {:content "."} :finish_reason nil}]}
+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"
+ :choices [{:index 0 :delta {} :finish_reason "stop"}]}])
+
+;;; The deprecated spelling, still emitted by older vLLM builds and other OpenAI-compatible servers.
+(def ^:private deprecated-reasoning-stream
+ (mapv (fn [chunk]
+ (update-in chunk [:choices 0 :delta]
+ (fn [delta]
+ (if-let [r (:reasoning delta)]
+ (-> delta (dissoc :reasoning) (assoc :reasoning_content r))
+ delta))))
+ reasoning-stream))
+
+(deftest ^:parallel reasoning-becomes-reasoning-parts-test
+ (testing "with --reasoning-parser, thinking streams as reasoning and the answer still streams as text"
+ (doseq [[spelling stream] {"reasoning" reasoning-stream
+ "reasoning_content" deprecated-reasoning-stream}]
+ (testing spelling
+ (let [parts (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)) stream)]
+ (is (= ["\nOkay, the user asks 2+2."] (map :text (filter #(= :reasoning (:type %)) parts))))
+ (is (= ["Four."] (map :text (filter #(= :text (:type %)) parts)))))))))
+
+(deftest ^:parallel reasoning-chunks-are-bracketed-test
+ (testing "the raw chunk stream brackets reasoning the way self.core expects to group it"
+ (let [chunks (into [] (vllm/vllm->aisdk-chunks-xf) reasoning-stream)
+ by-type (group-by :type chunks)]
+ (testing "the opening `content \"\"` chunk does not open a text block ahead of the reasoning"
+ (is (= [:start :reasoning-start :reasoning-delta :reasoning-delta :reasoning-end
+ :text-start :text-delta :text-delta :text-end]
+ (mapv :type chunks))))
+ (testing "every reasoning chunk shares one id, so aisdk-xf joins them into a single part"
+ (is (= 1 (count (into #{} (map :id) (mapcat by-type [:reasoning-start :reasoning-delta :reasoning-end])))))))))
+
+(deftest ^:parallel reasoning-forwarding-is-opt-in-test
+ (testing "the shared xf still drops reasoning for the adapters that did not opt in"
+ (doseq [[spelling stream] {"reasoning" reasoning-stream
+ "reasoning_content" deprecated-reasoning-stream}]
+ (testing spelling
+ (let [parts (into [] (comp (chat-completions/chat-completions->aisdk-chunks-xf) (self.core/aisdk-xf))
+ stream)]
+ (is (empty? (filter #(= :reasoning (:type %)) parts)))
+ (is (= ["Four."] (map :text (filter #(= :text (:type %)) parts)))))))))
+
+;;; Recorded verbatim from vLLM 0.26.0 serving vllm-test with
+;;; `--served-model-name vllm-test --max-model-len 32768 --enable-auto-tool-choice --tool-call-parser hermes --reasoning-parser qwen3`.
+;;; Inert envelope fields (created, logprobs, object, prompt_text, prompt_token_ids, service_tier, system_fingerprint, token_ids) dropped for size;
+;;; the full capture is under `logs/ai/requests/`.
+;;; 155 chunks captured, 22 kept: each run of reasoning deltas is truncated
+;;; to its first 2 (133 reasoning dropped). Nothing structural is removed.
+(def ^:private recorded-parallel-tool-call-stream
+ [{:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices [{:index 0, :delta {:role "assistant", :content ""}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices [{:index 0, :delta {:reasoning "\n"}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices [{:index 0, :delta {:reasoning "Okay"}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices [{:index 0, :delta {:content "\n\n"}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0,
+ :delta
+ {:content nil,
+ :tool_calls
+ [{:id "chatcmpl-tool-96102e0493e264b7", :type "function", :index 0, :function {:name "record_table_name"}}]},
+ :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "{\""}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "table"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "_name"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "\":"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments " \""}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "orders"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "\"}"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0,
+ :delta
+ {:content nil,
+ :tool_calls
+ [{:id "chatcmpl-tool-b34f3b547d2105f2", :type "function", :index 1, :function {:name "record_column_name"}}]},
+ :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "{\""}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "column"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "_name"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "\":"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments " \""}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "total"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices
+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "\"}"}}]}, :finish_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices [{:index 0, :delta {}, :finish_reason "tool_calls", :stop_reason nil}]}
+ {:id "chatcmpl-8ad4c404bbb917ae",
+ :model "vllm-test",
+ :choices [],
+ :usage {:prompt_tokens 244, :total_tokens 426, :completion_tokens 182}}])
+
+(deftest ^:parallel recorded-parallel-tool-calls-become-separate-parts-test
+ (testing "two tool calls in one assistant message accumulate into two parts, each with its own arguments"
+ (let [calls (->> recorded-parallel-tool-call-stream
+ (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)))
+ (filter #(= :tool-input (:type %))))]
+ (is (= [{:id "chatcmpl-tool-96102e0493e264b7" :function "record_table_name"
+ :arguments {:table_name "orders"}}
+ {:id "chatcmpl-tool-b34f3b547d2105f2" :function "record_column_name"
+ :arguments {:column_name "total"}}]
+ (mapv #(select-keys % [:id :function :arguments]) calls))))))
+
+(deftest ^:parallel recorded-parallel-tool-calls-do-not-interleave-test
+ (testing "the xf keys transitions on the tool-call id and never reads index, which is safe only because
+ hermes emits each call's deltas as one contiguous run"
+ (let [deltas (for [chunk recorded-parallel-tool-call-stream
+ choice (:choices chunk)
+ tc (get-in choice [:delta :tool_calls])]
+ tc)]
+ (testing "index arrives in contiguous runs, not interleaved"
+ (is (= [0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1] (mapv :index deltas))))
+ (testing "id arrives on each call's opening delta only — a provider repeating it would lose the
+ arguments, since neither the start branch nor the argument-delta branch would fire"
+ (is (= [0 1] (keep #(when (:id %) (:index %)) deltas)))))))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; Auth
+;;; ──────────────────────────────────────────────────────────────────
+
+(defn- captured-chat-request!
+ "Run `vllm-raw` against a stubbed transport and return the request it issued."
+ [opts]
+ (let [captured (atom nil)]
+ (with-redefs [self.core/sse-reducible (fn [_] (reify clojure.lang.IReduceInit
+ (reduce [_ _rf init] init)))
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] (reset! captured req) {:body nil})]
+ (vllm/vllm-raw opts))
+ @captured))
+
+(deftest vllm-auth-sends-bearer-token-when-configured-test
+ (testing "the connection's API key becomes an Authorization header against its base URL"
+ (is (=? {:method :post
+ :url "http://vllm.internal:8000/v1/chat/completions"
+ :headers {"Authorization" "Bearer local-dev-key"}
+ :body string?}
+ (captured-chat-request! {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials keyed-credentials})))))
+
+(deftest vllm-auth-omits-authorization-without-a-key-test
+ (testing "a keyless server is a complete configuration — no Authorization header, and no missing-key error"
+ (let [req (captured-chat-request! {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials credentials})]
+ (is (= "http://vllm.internal:8000/v1/chat/completions" (:url req)))
+ (is (not (contains? (:headers req) "Authorization"))))))
+
+(deftest vllm-sets-a-socket-timeout-test
+ (testing "generation requests carry the vLLM socket timeout and the shared connection timeout"
+ (mt/with-temporary-setting-values [llm.settings/llm-vllm-request-timeout-ms 300000]
+ (is (=? {:socket-timeout 300000
+ :connection-timeout (llm.settings/llm-connection-timeout-ms)}
+ (captured-chat-request! {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials credentials}))))))
+
+(deftest vllm-auth-missing-base-url-test
+ (testing "a connection carrying no base URL fails with a message naming vLLM, not a clj-http error about a
+ relative URL"
+ (with-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"No vLLM base URL is set"
+ (vllm/vllm-raw {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials {:api-key "local-dev-key"}})))
+ (is (= :base-url-missing
+ (:error-code (try (vllm/list-models {:credentials {:api-key "local-dev-key"}})
+ (catch clojure.lang.ExceptionInfo e (ex-data e)))))))))
+
+(deftest vllm-missing-model-test
+ (testing "a blank model fails with a vLLM-specific message before any request is made"
+ (with-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"No vLLM model is set"
+ (vllm/vllm-raw {:input [{:role :user :content "hi"}] :credentials credentials}))))))
+
+(deftest vllm-ai-proxy-unsupported-test
+ (testing "ai-proxy? throws before credentials are even consulted"
+ (with-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"AI proxy is not supported for vLLM"
+ (vllm/vllm-raw {:model "vllm-test" :input [{:role :user :content "hi"}] :ai-proxy? true})))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"AI proxy is not supported for vLLM"
+ (vllm/list-models {:ai-proxy? true}))))))
+
+(deftest vllm-stream-socket-timeout-is-not-retryable-test
+ (testing "a socket timeout while consuming the stream surfaces as a tagged vLLM error, not a raw SocketTimeoutException"
+ (with-redefs [self.core/sse-reducible (fn [_]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ _rf _init]
+ (throw (SocketTimeoutException. "Read timed out")))))
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [_] {:body nil})]
+ (let [e (try (into [] (vllm/vllm-raw {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials credentials}))
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :vllm-timeout (:error-code (ex-data e))))
+ (is (re-find #"stopped responding" (ex-message e)))
+ (testing "and is not retryable"
+ (is (false? (#'self/retryable-error? e))))))))
+
+(deftest vllm-stream-connection-failure-is-not-retryable-test
+ (testing "a stream severed mid-body surfaces as a tagged vLLM error, not a raw IOException"
+ (with-redefs [self.core/sse-reducible (fn [_]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ _rf _init]
+ (throw (java.io.IOException. "Connection reset")))))
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [_] {:body nil})]
+ (let [e (try (into [] (vllm/vllm-raw {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials credentials}))
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :vllm-stream-interrupted (:error-code (ex-data e))))
+ (is (re-find #"interrupted before the response finished" (ex-message e)))
+ (testing "and is not retryable"
+ (is (false? (#'self/retryable-error? e))))))))
+
+(deftest vllm-mid-stream-non-io-error-is-translated-test
+ (testing "a mid-stream failure that is not IO gets the shared provider translation, the same as every
+ other adapter — `io-guarded` composes with `reducible-with-api-errors` rather than replacing it"
+ (with-redefs [self.core/sse-reducible (fn [_]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ _rf _init]
+ (throw (ex-info "clj-http: status 500"
+ {:status 500 :body "{\"message\":\"boom\"}"})))))
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [_] {:body nil})]
+ (let [e (try (into [] (vllm/vllm-raw {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials credentials}))
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :provider-api-error (:error-code (ex-data e))))
+ (is (re-find #"vLLM returned an internal server error" (ex-message e)))))))
+
+(deftest vllm-request-timeout-names-vllm-and-its-setting-test
+ (testing "a timeout while establishing the request gets the same treatment the preflight already gives it,
+ rather than `rethrow-api-error!`'s \"vllm API request failed: Read timed out\""
+ (mt/with-temporary-setting-values [llm.settings/llm-vllm-request-timeout-ms 300000]
+ (with-redefs [http/request (fn [_] (throw (SocketTimeoutException. "Read timed out")))]
+ (let [e (try (vllm/vllm-raw {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials credentials})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :vllm-timeout (:error-code (ex-data e))))
+ (is (re-find #"did not respond within 300000ms" (ex-message e)))
+ (is (re-find #"raise the vLLM request timeout" (ex-message e)))
+ (testing "and is not retryable"
+ (is (false? (#'self/retryable-error? e)))))))))
+
+(deftest vllm-unreachable-server-names-the-base-url-test
+ (testing "a refused connection is the base URL being wrong or the server being down — say which URL failed"
+ (with-redefs [http/request (fn [_] (throw (ConnectException. "Connection refused")))]
+ (let [e (try (vllm/vllm-raw {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials credentials})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :vllm-unreachable (:error-code (ex-data e))))
+ (is (re-find #"Could not reach the vLLM server at http://vllm\.internal:8000/v1" (ex-message e)))
+ (testing "and is not retryable"
+ (is (false? (#'self/retryable-error? e))))))))
+
+(deftest vllm-http-errors-still-reach-the-status-specific-message-test
+ (testing "the IOException catch runs first, so a non-2xx must still be translated by `vllm-error-msg`"
+ (with-redefs [http/request (fn [_] (throw (ex-info "clj-http: status 401"
+ {:status 401 :body "{\"message\":\"Unauthorized\"}"})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"vLLM API key expired or invalid"
+ (vllm/vllm-raw {:model "vllm-test"
+ :input [{:role :user :content "hi"}]
+ :credentials credentials}))))))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; list-models
+;;; ──────────────────────────────────────────────────────────────────
+
+;;; Recorded from vLLM 0.26.0 serving mlx-community/Qwen3-14B-4bit with
+;;; `--served-model-name vllm-test vllm-test-alias qwen3-14b` (BOT-1930 L-10). Aliases are the cheapest
+;;; multi-entry catalog: several ids over one loaded model. Note the key set — there is no `name`, so
+;;; `display_name` always falls back to the served id on vLLM, and `parent` is present-and-nil rather
+;;; than absent. `:permission` is dropped here; nothing reads it.
+(def ^:private recorded-multi-model-catalog
+ [{:id "vllm-test" :object "model" :created 1786676106 :owned_by "vllm"
+ :root "mlx-community/Qwen3-14B-4bit" :parent nil :max_model_len 32768}
+ {:id "vllm-test-alias" :object "model" :created 1786676106 :owned_by "vllm"
+ :root "mlx-community/Qwen3-14B-4bit" :parent nil :max_model_len 32768}
+ {:id "qwen3-14b" :object "model" :created 1786676106 :owned_by "vllm"
+ :root "mlx-community/Qwen3-14B-4bit" :parent nil :max_model_len 32768}])
+
+(deftest list-models-passes-the-catalog-through-test
+ (testing "every served model is offered — there is no whitelist — in the order the server lists them"
+ (mt/with-dynamic-fn-redefs [http/request (fn [req]
+ (is (=? {:method :get
+ :url "http://vllm.internal:8000/v1/models"
+ :headers {"Authorization" "Bearer local-dev-key"}}
+ req))
+ {:status 200 :body {:data recorded-multi-model-catalog}})]
+ (is (= {:models [{:id "vllm-test" :display_name "vllm-test"}
+ {:id "vllm-test-alias" :display_name "vllm-test-alias"}
+ {:id "qwen3-14b" :display_name "qwen3-14b"}]}
+ (vllm/list-models {:credentials keyed-credentials}))))))
+
+;;; Recorded from the same server started **without** `--served-model-name` (BOT-1930 L-10), which is
+;;; how an operator who does not rename their deployment sees it: the catalog id is the Hugging Face
+;;; repo id, slashes and all, and `root` equals `id` rather than sitting behind an alias.
+(def ^:private recorded-no-alias-catalog
+ [{:id "mlx-community/Qwen3-14B-4bit" :object "model" :created 1786676633 :owned_by "vllm"
+ :root "mlx-community/Qwen3-14B-4bit" :parent nil :max_model_len 32768}])
+
+(deftest list-models-keeps-slashes-in-a-served-model-id-test
+ (testing "a Hugging Face repo id survives the listing intact — `llm-metabot-provider` stores it as
+ `vllm/{id}`, so the model segment is the only one allowed to keep its slashes"
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body {:data recorded-no-alias-catalog}})]
+ (is (= {:models [{:id "mlx-community/Qwen3-14B-4bit" :display_name "mlx-community/Qwen3-14B-4bit"}]}
+ (vllm/list-models {:credentials credentials}))))))
+
+(deftest list-models-display-name-falls-back-to-the-served-id-test
+ (testing "a `name` is honoured when present, though no vLLM build emits one — the tolerance is for the
+ other OpenAI-compatible servers the same UI copy invites"
+ (mt/with-dynamic-fn-redefs [http/request (fn [_]
+ {:status 200
+ :body {:data [{:id "some-finetune" :name "Some Finetune"}]}})]
+ (is (= {:models [{:id "some-finetune" :display_name "Some Finetune"}]}
+ (vllm/list-models {:credentials credentials}))))))
+
+(deftest list-models-does-not-fall-back-to-the-single-provider-settings-test
+ (testing "the connection's credentials are the only source — a setting configuring another connection
+ is not a fallback"
+ (mt/with-temporary-setting-values [llm.settings/llm-vllm-api-base-url "http://saved:8000/v1"
+ llm.settings/llm-vllm-api-key "saved-key"]
+ (mt/with-dynamic-fn-redefs [http/request (fn [req]
+ (is (=? {:url "http://explicit:8000/v1/models"
+ :headers {"Authorization" "Bearer explicit-key"}}
+ req))
+ {:status 200 :body {:data []}})]
+ (is (= {:models []}
+ (vllm/list-models {:credentials {:base-url "http://explicit:8000/v1"
+ :api-key "explicit-key"}}))))
+ (testing "and a connection without a base URL fails rather than borrowing the setting's"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"No vLLM base URL is set"
+ (vllm/list-models {:credentials {:api-key "explicit-key"}})))))))
+
+(deftest list-models-fails-closed-on-a-body-that-is-not-a-catalog-test
+ (testing "a 2xx whose body carries no model list throws, naming the base URL"
+ (doseq [body [{:status "ok" :service "some-other-thing"} {:object "list"} "<html>404</html>"]]
+ (testing (str "body " (pr-str body))
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body body})]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"vLLM returned an unexpected model list response.*http://vllm\.internal:8000/v1"
+ (vllm/list-models {:credentials credentials})))))))
+ (testing "a well-formed but empty catalog lists nothing and gets its own distinct message"
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body {:object "list" :data []}})]
+ (is (= {:models []} (vllm/list-models {:credentials credentials})))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"reachable but is not serving any models"
+ (vllm/list-models {:credentials credentials :probe? true}))))))
+
+(deftest list-models-fails-closed-before-probing-test
+ (testing "a malformed catalog throws without issuing a probe request"
+ (let [chat-requests (atom 0)]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]
+ (when-not (re-find #"/models$" (str url))
+ (swap! chat-requests inc))
+ {:status 200 :body {:status "ok"}})]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"vLLM returned an unexpected model list response"
+ (vllm/list-models {:credentials credentials :probe? true})))
+ (is (zero? @chat-requests))))))
+
+(deftest list-models-401-maps-to-invalid-key-message-test
+ (testing "a 401 surfaces as the canonical message naming --api-key"
+ (mt/with-dynamic-fn-redefs [http/request (fn [_]
+ (throw (ex-info "clj-http: status 401"
+ {:status 401
+ :body "{\"message\":\"Unauthorized\"}"})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"vLLM API key expired or invalid"
+ (vllm/list-models {:credentials (assoc credentials :api-key "wrong-key")}))))))
+
+(deftest list-models-404-points-at-the-base-url-test
+ (testing "a 404 tells the admin the base URL should end in /v1"
+ (mt/with-dynamic-fn-redefs [http/request (fn [_]
+ (throw (ex-info "clj-http: status 404" {:status 404 :body "not found"})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"base URL should end in /v1"
+ (vllm/list-models {:credentials {:base-url "http://vllm.internal:8000"}}))))))
+
+(deftest list-models-unreachable-server-is-a-client-error-test
+ (testing "a refused connection while fetching the catalog names the base URL and is tagged 400, so a
+ mistyped base URL reaches the admin as the message instead of a 500"
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (ConnectException. "Connection refused")))]
+ (let [e (try (vllm/list-models {:credentials credentials})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= {:api-error true :status-code 400 :error-code :vllm-unreachable}
+ (select-keys (ex-data e) [:api-error :status-code :error-code])))
+ (is (re-find #"Could not reach the vLLM server at http://vllm\.internal:8000/v1" (ex-message e)))))))
+
+(deftest list-models-timeout-names-the-server-test
+ (testing "the catalog fetch runs on the shared request timeout, so a stall there reports that budget
+ rather than the vLLM generation one"
+ (mt/with-temporary-setting-values [llm.settings/llm-request-timeout-ms 120000
+ llm.settings/llm-vllm-request-timeout-ms 300000]
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (SocketTimeoutException. "Read timed out")))]
+ (let [e (try (vllm/list-models {:credentials credentials})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= {:api-error true :status-code 400 :error-code :vllm-timeout}
+ (select-keys (ex-data e) [:api-error :status-code :error-code])))
+ (is (re-find #"http://vllm\.internal:8000/v1 did not respond within 120000ms" (ex-message e))))))))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; Preflight
+;;; ──────────────────────────────────────────────────────────────────
+
+(defn- probing-server
+ "Stub `http/request` for the preflight path: `GET /models` returns `models`, and each
+ `POST /chat/completions` returns whatever `choice-by-tool-choice` holds for the `tool_choice` it
+ was sent.
+
+ Dispatching on `tool_choice` is what lets the two probes disagree. They exercise different server
+ capabilities — the tool-call parser under `auto`, guided decoding under `required` — so a stub that
+ answers both identically can only ever reach `check-structured-output!` on the happy path."
+ [models choice-by-tool-choice]
+ (fn [{:keys [url body]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data models}}
+ (let [tool-choice (:tool_choice (json/decode+kw (str body)))]
+ {:status 200 :body {:choices [(get choice-by-tool-choice tool-choice)]}}))))
+
+(def ^:private tool-calling-message
+ {:content ""
+ :tool_calls [{:id "chatcmpl-tool-1"
+ :type "function"
+ :function {:name "record_table_name" :arguments "{\"table_name\": \"orders\"}"}}]})
+
+(defn- probe-choice!
+ "Run a preflight against a stub. The 2-arity answers both probes alike; the 3-arity lets them differ."
+ ([models chat-choice]
+ (probe-choice! models chat-choice chat-choice))
+ ([models auto-choice required-choice]
+ (mt/with-dynamic-fn-redefs [http/request (probing-server models {"auto" auto-choice
+ "required" required-choice})]
+ (vllm/list-models {:credentials credentials :probe? true}))))
+
+(defn- probe!
+ "[[probe-choice!]] for a server that runs to a natural stop, where only the message shape matters."
+ [models chat-message]
+ (probe-choice! models {:message chat-message :finish_reason "tool_calls"}))
+
+(deftest preflight-passes-on-a-correctly-configured-server-test
+ (testing "a server that returns a well-formed tool call passes and still lists its models"
+ (is (= {:models [{:id "vllm-test" :display_name "vllm-test"}]
+ :probed-model "vllm-test"
+ :learned-config {vllm/reasoning-config-key "false"}}
+ (probe! [{:id "vllm-test" :max_model_len 32768}] tool-calling-message)))))
+
+(deftest preflight-reports-the-model-it-probed-test
+ (testing "the probed model is named in the result rather than left to be re-derived from the listing —
+ the connect path must adopt the model the contract checks actually ran against"
+ (testing "the first catalog entry, in the order the server lists it"
+ (is (= "vllm-test"
+ (:probed-model (probe! recorded-multi-model-catalog tool-calling-message)))))
+ (testing "including a slash-bearing repo id, which the connect path stores as `vllm/{id}`"
+ (is (= "mlx-community/Qwen3-14B-4bit"
+ (:probed-model (probe! recorded-no-alias-catalog tool-calling-message))))))
+ (testing "and a listing that did not probe reports nothing"
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body {:data [{:id "vllm-test"}]}})]
+ (is (= [:models] (keys (vllm/list-models {:credentials credentials})))))))
+
+(deftest preflight-skips-lora-adapters-when-picking-a-default-test
+ (testing "an adapter registered with --lora-modules carries `parent`; adopting one would point Metabot
+ at an adapter the admin never chose"
+ (let [probed (atom nil)]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200
+ :body {:data [{:id "sql-lora"
+ :parent "vllm-test"
+ :max_model_len 32768}
+ {:id "vllm-test"
+ :parent nil
+ :max_model_len 32768}]}}
+ (do (reset! probed (:model (json/decode+kw (str body))))
+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]
+ (is (= "vllm-test" (:probed-model (vllm/list-models {:credentials credentials :probe? true}))))
+ (is (= "vllm-test" @probed)))))
+ (testing "an explicitly requested adapter is still honoured — the filter only picks the default"
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200
+ :body {:data [{:id "sql-lora" :parent "vllm-test" :max_model_len 32768}
+ {:id "vllm-test" :max_model_len 32768}]}}
+ {:status 200 :body {:choices [{:message tool-calling-message}]}}))]
+ (is (= "sql-lora" (:probed-model (vllm/list-models {:credentials credentials
+ :probe? true
+ :model "sql-lora"})))))))
+
+(deftest preflight-skipped-without-probe-flag-test
+ (testing "without :probe? no generation request is made at all — listing models must stay cheap"
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]
+ (when-not (re-find #"/models$" (str url))
+ (throw (ex-info "preflight must not run on a plain listing" {})))
+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}})]
+ (is (= {:models [{:id "vllm-test" :display_name "vllm-test"}]}
+ (vllm/list-models {:credentials credentials}))))))
+
+(deftest preflight-rejects-a-prose-answer-test
+ (testing "a server that answers with text instead of a tool call names the two flags that cause it"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"--enable-auto-tool-choice"
+ (probe! [{:id "vllm-test" :max_model_len 32768}]
+ {:content "I'll record the table name orders for you." :tool_calls []})))))
+
+(deftest preflight-rejects-unparseable-tool-arguments-test
+ (testing "a tool call whose arguments are not JSON points at a mismatched --tool-call-parser"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"--tool-call-parser"
+ (probe! [{:id "vllm-test" :max_model_len 32768}]
+ {:content ""
+ :tool_calls [{:id "chatcmpl-tool-1"
+ :type "function"
+ :function {:name "record_table_name" :arguments "table_name=orders"}}]})))))
+
+(deftest preflight-rejects-leaked-reasoning-test
+ (testing "reasoning arriving as chat text points at --reasoning-parser, even when the tool call itself is fine"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"--reasoning-parser"
+ (probe! [{:id "vllm-test" :max_model_len 32768}]
+ (assoc tool-calling-message :content "<think>\nThe user wants orders.\n</think>\n\n"))))))
+
+(deftest preflight-rejects-a-context-window-that-is-too-small-test
+ (testing "a model served under the context floor is rejected before any generation request"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"too small for Metabot"
+ (probe! [{:id "vllm-test" :max_model_len 4096}] tool-calling-message)))))
+
+(deftest preflight-rejects-a-server-with-no-models-test
+ (testing "a reachable server serving nothing is a configuration error, not an empty dropdown"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"not serving any models"
+ (probe! [] tool-calling-message)))))
+
+(deftest preflight-probes-the-requested-model-test
+ (testing "the probe targets the requested model, not merely the first one served"
+ (let [probed (atom nil)]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data [{:id "first" :max_model_len 32768}
+ {:id "second" :max_model_len 32768}]}}
+ (do (reset! probed (re-find #"second" (str body)))
+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]
+ (vllm/list-models {:credentials credentials :probe? true :model "second"})
+ (is (= "second" @probed))))))
+
+(deftest preflight-rejects-a-model-the-server-does-not-serve-test
+ (testing "a requested model absent from the catalog fails and names what is served, rather than probing something else"
+ (let [generated? (atom false)]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data [{:id "served-a" :max_model_len 32768}
+ {:id "served-b" :max_model_len 32768}]}}
+ (do (reset! generated? true)
+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"not serving missing-model. It is serving: served-a, served-b"
+ (vllm/list-models {:credentials credentials :probe? true :model "missing-model"})))
+ (is (false? @generated?)
+ "no generation request is issued for a model the server cannot run")))))
+
+(deftest preflight-reports-a-thinking-budget-overrun-test
+ (testing "a reasoning model that never stops thinking is named as such, not as a server missing tool-calling flags"
+ (let [e (try (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
+ {:message {:content nil
+ :reasoning "Okay, the user wants me to record a table name, so"
+ :tool_calls []}
+ :finish_reason "length"})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (re-find #"reasoning without calling a tool" (ex-message e)))
+ (is (not (re-find #"--enable-auto-tool-choice" (ex-message e)))
+ "the flags this server needs are already set, so naming them would send the admin the wrong way"))))
+
+;;; The `content` below is the shape a live vLLM 0.26.0 returns for a tool call cut off at
+;;; `max_tokens`, serving mlx-community/Qwen3-14B-4bit with `--enable-auto-tool-choice
+;;; --tool-call-parser hermes` and `chat_template_kwargs {enable_thinking false}` (BOT-1930 L-7).
+;;; The parser extracts from complete output, so without its closing `</tool_call>` the call yields no
+;;; `tool_calls` entry at all and the raw text stays in `content`.
+(deftest preflight-reports-a-truncated-tool-call-as-a-ceiling-overrun-test
+ (testing "a generation cut off at the ceiling names the ceiling rather than the tool-calling flags,
+ which the unterminated sentinel in the content shows are working"
+ (let [e (try (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
+ {:message {:content "<tool_call>\n{\"name\": \"record_table_name\", \"arguments\": {\""
+ :tool_calls []}
+ :finish_reason "length"})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (re-find #"before completing a tool call" (ex-message e)))
+ (is (re-find #"2048 token" (ex-message e)))
+ (is (not (re-find #"--enable-auto-tool-choice" (ex-message e)))))))
+
+(deftest preflight-reports-truncated-prose-as-a-ceiling-overrun-test
+ (testing "prose truncated at the ceiling gets the same verdict — finish_reason is the only signal
+ separating it from a truncated tool call"
+ (let [e (try (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
+ {:message {:content "Sure! To record the table name I would first need to"
+ :tool_calls []}
+ :finish_reason "length"})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (re-find #"before completing a tool call" (ex-message e))))))
+
+(deftest preflight-probe-timeout-is-capped-test
+ (testing "a probe runs on its own budget, capped below the inference timeout"
+ (mt/with-temporary-setting-values [llm.settings/llm-vllm-request-timeout-ms 600000]
+ (let [socket-timeouts (atom [])]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url socket-timeout]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}
+ (do (swap! socket-timeouts conj socket-timeout)
+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]
+ (vllm/list-models {:credentials credentials :probe? true})
+ (is (= #{120000} (set @socket-timeouts))))))))
+
+(deftest preflight-surfaces-a-probe-timeout-test
+ (testing "a probe that times out says the server is too slow, not that the request failed"
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}
+ (throw (SocketTimeoutException. "Read timed out"))))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"did not answer the connection test"
+ (vllm/list-models {:credentials credentials :probe? true}))))))
+
+(deftest preflight-probes-run-concurrently-test
+ (testing "both probes are in flight at once, so a slow server costs one probe budget rather than two"
+ (let [arrived (CountDownLatch. 2)]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}
+ (do (.countDown arrived)
+ ;; Only satisfiable if the other probe is
+ ;; already in flight.
+ (is (.await arrived 10 TimeUnit/SECONDS))
+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]
+ (vllm/list-models {:credentials credentials :probe? true})))))
+
+(deftest preflight-reports-whether-the-probed-model-reasons-test
+ (testing "the probe is the only place this is knowable — /v1/models carries no reasoning field, so what it
+ saw is reported back for the connection to record"
+ ;; Recorded shape: vLLM 0.26 puts it on the non-streaming message under `reasoning`.
+ (is (= {vllm/reasoning-config-key "true"}
+ (:learned-config
+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
+ {:message (assoc tool-calling-message
+ :reasoning "\nOkay, the user wants me to record \"orders\".")
+ :finish_reason "tool_calls"})))))
+ (testing "and under the deprecated spelling older servers still use"
+ (is (= {vllm/reasoning-config-key "true"}
+ (:learned-config
+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
+ {:message (assoc tool-calling-message
+ :reasoning_content "Older vLLM builds spell it this way.")
+ :finish_reason "tool_calls"})))))
+ (testing "a model that answers without reasoning reports false"
+ (is (= {vllm/reasoning-config-key "false"}
+ (:learned-config (probe! [{:id "vllm-test" :max_model_len 32768}] tool-calling-message)))))
+ (testing "and a connection carrying the flag is what the request body reads it from"
+ (is (true? (vllm/reasoning-connection? {vllm/reasoning-config-key "true"})))
+ (is (false? (vllm/reasoning-connection? {vllm/reasoning-config-key "false"})))
+ (is (false? (vllm/reasoning-connection? {})))
+ (testing "including the JSON boolean a hand-written llm-providers can hold"
+ (is (true? (vllm/reasoning-connection? {vllm/reasoning-config-key true}))))))
+
+(deftest preflight-reports-nothing-when-it-fails-test
+ (testing "a failed probe throws rather than reporting a guess, so the connection records nothing"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"--enable-auto-tool-choice"
+ (probe! [{:id "vllm-test" :max_model_len 32768}]
+ {:content "I'll record the table name orders for you." :tool_calls []})))))
+
+(deftest preflight-failures-are-surfaced-as-client-errors-test
+ (testing "a preflight failure is tagged 400 so the admin sees the message instead of a 500"
+ (let [data (try (probe! [{:id "vllm-test" :max_model_len 4096}] tool-calling-message)
+ (catch clojure.lang.ExceptionInfo e (ex-data e)))]
+ (is (= {:api-error true :status-code 400 :error-code :vllm-preflight-failed}
+ (select-keys data [:api-error :status-code :error-code]))))))
+
+(deftest preflight-rejects-a-server-that-cannot-force-a-tool-call-test
+ (testing "a server whose --tool-call-parser is right but whose model cannot compile a grammar answers
+ `auto` correctly and `required` with prose — ordinary chat looks fine while conversation
+ titling and the whole sql profile break"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"did not honor a forced tool call"
+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
+ {:message tool-calling-message :finish_reason "tool_calls"}
+ {:message {:content "I can record the table name for you." :tool_calls []}
+ :finish_reason "stop"})))))
+
+(deftest preflight-reports-a-forced-tool-call-that-ran-out-of-budget-test
+ (testing "a forced call truncated at the ceiling is named as such, not as a server refusing to honor it"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"reached the 2048 token connection-test ceiling"
+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
+ {:message tool-calling-message :finish_reason "tool_calls"}
+ {:message {:content "" :tool_calls []}
+ :finish_reason "length"})))))
+
+(deftest preflight-cancels-the-sibling-probe-on-failure-test
+ (testing "the first verdict returns immediately and its sibling is cancelled — an abandoned future would
+ keep generating against the operator's server after the admin already has a 400"
+ (let [interrupted (promise)
+ never (CountDownLatch. 1)]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}
+ (case (:tool_choice (json/decode+kw (str body)))
+ "auto" {:status 200
+ :body {:choices [{:message {:content "I'll record orders."
+ :tool_calls []}}]}}
+ "required" (try
+ (.await never 10 TimeUnit/SECONDS)
+ (deliver interrupted false)
+ {:status 200 :body {:choices [{:message tool-calling-message}]}}
+ (catch InterruptedException _
+ (deliver interrupted true)
+ (throw (SocketTimeoutException. "interrupted")))))))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"--enable-auto-tool-choice"
+ (vllm/list-models {:credentials credentials :probe? true})))
+ (is (true? (deref interrupted 10000 :never-cancelled)))))))
+
+(deftest preflight-probe-request-shape-test
+ (testing "both probes ask for the same trivial completion, non-streaming and deterministic — the ceiling
+ is the one a forced tool call is guaranteed to be given at request time"
+ (let [requests (atom [])]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body] :as req}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}
+ (do (swap! requests conj (assoc req :decoded (json/decode+kw (str body))))
+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]
+ (vllm/list-models {:credentials credentials :probe? true})
+ (is (= 2 (count @requests)))
+ (is (= #{"auto" "required"} (set (map #(-> % :decoded :tool_choice) @requests))))
+ (doseq [{:keys [decoded] :as req} @requests]
+ (is (= :post (:method req)))
+ (is (= "http://vllm.internal:8000/v1/chat/completions" (:url req)))
+ (is (= :json (:as req))
+ "a probe reads a whole response; :stream would leave the verdict unreadable")
+ (is (nil? (:stream decoded)))
+ (is (= 0 (:temperature decoded)))
+ (is (= 2048 (:max_tokens decoded))
+ "must not drift from the floor a forced tool call is raised to — the preflight's promise is
+ that a model which connected can already clear the budget it will be run at")
+ (is (= 1 (count (:tools decoded))))
+ (is (= "record_table_name" (-> decoded :tools first :function :name))))))))
+
+(deftest preflight-surfaces-an-http-error-as-a-provider-error-test
+ (testing "a 400 from /chat/completions is the server rejecting the request, not a contract failure —
+ it must keep vLLM's own status text rather than being reworded as a preflight verdict"
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}
+ (throw (ex-info "clj-http: status 400"
+ {:status 400
+ :body "{\"message\":\"cannot compile grammar\"}"}))))]
+ (let [e (try (vllm/list-models {:credentials credentials :probe? true})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (re-find #"vLLM rejected the request" (ex-message e)))
+ (is (= :provider-api-error (:error-code (ex-data e))))))))
+
+(deftest preflight-accepts-a-catalog-without-a-context-window-test
+ (testing "`max_model_len` is vLLM's own field: Ollama, LM Studio, and TGI omit it. The floor is
+ best-effort, so a catalog without it connects rather than being rejected on a missing field"
+ (is (= {:models [{:id "vllm-test" :display_name "vllm-test"}]
+ :probed-model "vllm-test"
+ :learned-config {vllm/reasoning-config-key "false"}}
+ (probe! [{:id "vllm-test"}] tool-calling-message)))))
+
+(deftest preflight-does-not-leak-the-context-window-to-the-client-test
+ (testing "`max_model_len` feeds the context check and stops there — the model list the admin sees carries
+ only what the dropdown renders"
+ (let [models (:models (probe! [{:id "vllm-test" :max_model_len 32768 :parent nil :root "org/Model"}]
+ tool-calling-message))]
+ (is (= [{:id "vllm-test" :display_name "vllm-test"}] models)))))
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index e9be9153967..172ba28275d 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -74,6 +74,10 @@
(#'self/parse-provider-model "moonshot/kimi-k3")))
(is (=? {:provider "google" :model "google/gemini-3.5-flash" :ai-proxy? false}
(#'self/parse-provider-model "google/google/gemini-3.5-flash"))))
+ (testing "a vLLM served model is often a Hugging Face repo id, so the model segment keeps its slashes"
+ (llm.tu/with-connections [(llm.tu/connection "vllm")]
+ (is (=? {:provider "vllm" :model "mlx-community/Qwen3-14B-4bit" :ai-proxy? false}
+ (#'self/parse-provider-model "vllm/mlx-community/Qwen3-14B-4bit")))))
(testing "serves the managed connection through the wire family the model names"
(is (=? {:provider "anthropic" :model "claude-haiku-4-5" :ai-proxy? true}
(#'self/parse-provider-model "metabase/anthropic/claude-haiku-4-5")))
@@ -94,7 +98,8 @@
(is (fn? (#'self/resolve-adapter "zai")))
(is (fn? (#'self/resolve-adapter "mistral")))
(is (fn? (#'self/resolve-adapter "moonshot")))
- (is (fn? (#'self/resolve-adapter "google"))))
+ (is (fn? (#'self/resolve-adapter "google")))
+ (is (fn? (#'self/resolve-adapter "vllm"))))
(testing "throws for unknown provider"
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unknown LLM provider"
(#'self/resolve-adapter "unknown")))))
@@ -969,7 +974,17 @@
;; but other stuff is not
false (RuntimeException. "oops")
;; a wrapped non-transient cause stays non-retryable
- false (ex-info "boom" {} (IllegalArgumentException. "bad")))))
+ false (ex-info "boom" {} (IllegalArgumentException. "bad"))))
+ (testing ":retryable? false outranks both the status check and the cause walk"
+ (are [x y] (= x (#'self/retryable-error? y))
+ false (ex-info "vLLM stopped responding"
+ {:api-error true :error-code :vllm-timeout :retryable? false}
+ (java.net.SocketTimeoutException. "Read timed out"))
+ false (ex-info "vLLM queue full" {:status 429 :retryable? false})))
+ (testing "the tag is an opt-out only — :retryable? true cannot force a retry"
+ (are [x y] (= x (#'self/retryable-error? y))
+ true (ex-info "rate limited" {:status 429 :retryable? true})
+ false (ex-info "bad request" {:status 400 :retryable? true}))))
(deftest retry-delay-ms-test
(testing "backoff"
diff --git a/test/metabase/metabot/settings_test.clj b/test/metabase/metabot/settings_test.clj
index 9b3b1cb32ab..b53e152c817 100644
--- a/test/metabase/metabot/settings_test.clj
+++ b/test/metabase/metabot/settings_test.clj
@@ -195,6 +195,39 @@
(with-selected-model model-ref
(is (= expected (metabot.settings/llm-metabot-supports-reasoning?)))))))))
+(deftest metabot-supports-reasoning-vllm-test
+ (testing "vLLM answers from what the connect-time probe recorded on the connection, since neither its catalog
+ nor its model names carry a reasoning field"
+ (doseq [recorded ["true" "false"]]
+ (testing (str "recorded " recorded)
+ (with-connections [(connection "vllm" "vllm" {:base-url "http://vllm.internal:8000/v1"
+ :model-reasoning recorded})]
+ (with-selected-model "vllm/vllm-test"
+ (is (= (= "true" recorded) (metabot.settings/llm-metabot-supports-reasoning?))))))))
+ (testing "an unprobed server defaults to the non-reasoning renderer rather than guessing"
+ (with-connections [(connection "vllm" "vllm" {:base-url "http://vllm.internal:8000/v1"})]
+ (with-selected-model "vllm/vllm-test"
+ (is (false? (metabot.settings/llm-metabot-supports-reasoning?)))))))
+
+(deftest metabot-configured-with-a-keyless-vllm-connection-test
+ (testing "a vLLM server started without --api-key is a complete configuration: the base URL is the credential"
+ (with-connections [(connection "vllm" "vllm" {:base-url "http://vllm.internal:8000/v1"})]
+ (with-selected-model "vllm/vllm-test"
+ (is (true? (metabot.settings/llm-metabot-configured?))))))
+ (testing "and a connection carrying only a key is not"
+ (with-connections [(connection "vllm" "vllm" {:api-key "local-dev-key"})]
+ (with-selected-model "vllm/vllm-test"
+ (is (false? (metabot.settings/llm-metabot-configured?)))))))
+
+(deftest metabot-provider-keeps-slashes-in-a-vllm-model-test
+ (testing "a served model named after its Hugging Face repo keeps its slashes — everything after the first
+ segment is the model"
+ (with-connections [(connection "vllm" "vllm" {:base-url "http://vllm.internal:8000/v1"})]
+ (mt/discard-setting-changes [llm-metabot-provider]
+ (metabot.settings/llm-metabot-provider! "vllm/mlx-community/Qwen3-14B-4bit")
+ (is (=? {:provider "vllm" :model "mlx-community/Qwen3-14B-4bit"}
+ (#'metabot.self/parse-provider-model (metabot.settings/llm-metabot-provider))))))))
+
;;; ------------------------------------------- validate-metabot-provider! Tests -------------------------------------------
;; The validator is private; exercise it through the setting setter.