The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[BOT-1930] Self hosted vLLM metabot support (#79437)
Tyler Plude8661fd452421
diff --git a/frontend/src/metabase-types/api/llm.ts b/frontend/src/metabase-types/api/llm.tsindex 0b77b151b88..3fea04e89e8 100644--- a/frontend/src/metabase-types/api/llm.ts+++ b/frontend/src/metabase-types/api/llm.ts@@ -43,6 +43,7 @@ export type LlmProviderTypeName =| "google"| "azure"| "bedrock"+ | "vllm"| "metabase";export type LlmProviderFieldType =diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.tsindex a2546644b49..5af7fbc9a17 100644--- a/frontend/src/metabase-types/api/settings.ts+++ b/frontend/src/metabase-types/api/settings.ts@@ -542,6 +542,8 @@ interface SettingsManagerSettings {"llm-bedrock-secret-access-key"?: string | null;"llm-bedrock-region"?: string | null;"llm-bedrock-session-token"?: string | null;+ "llm-vllm-api-base-url"?: string | null;+ "llm-vllm-api-key"?: string | null;"openai-api-key": string | null;"openai-available-models"?: OpenAiModel[];"openai-model": string | null;diff --git a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.tsx b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.tsxindex 641a20f52bb..ad9ad48cec7 100644--- a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.tsx+++ b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.tsx@@ -19,7 +19,8 @@ const GENERIC_PROVIDER_ICON = "ai";// Every mark is inlined rather than loaded by URL: it lets the single-colour ones take the theme's text// colour the way their brands publish them, and it keeps them out of the emitted-asset pipeline. Metabase-// is null because it is drawn from its own logo component.+// is null because it is drawn from its own logo component, and vLLM because it falls back to the generic+// AI icon.const PROVIDER_LOGOS: Record<LlmProviderTypeName,ComponentType<SVGProps<SVGSVGElement>> | null@@ -33,6 +34,7 @@ const PROVIDER_LOGOS: Record<moonshot: MoonshotMark,google: GoogleMark,azure: AzureMark,+ vllm: null,metabase: null,};diff --git a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.unit.spec.tsx b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.unit.spec.tsxindex ff40f74fc95..a05399ea6b0 100644--- a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.unit.spec.tsx+++ b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderTypeIcon.unit.spec.tsx@@ -39,6 +39,13 @@ describe("ProviderTypeIcon", () => {expect(screen.getByRole("presentation")).toBeInTheDocument();});+ it("falls back to a generic icon for vLLM, which is a server rather than a vendor", () => {+ renderWithProviders(<ProviderTypeIcon type="vllm" />);++ expect(screen.queryByTestId("main-logo")).not.toBeInTheDocument();+ expect(screen.queryByRole("presentation")).not.toBeInTheDocument();+ });+it("falls back to a generic icon for a type the frontend does not know yet", () => {// the registry is the backend's, so it can serve a type this union does not list yet;// the cast reproduces that server response, which is unreachable through the type alonediff --git a/src/metabase/llm/api/provider.clj b/src/metabase/llm/api/provider.cljindex 4a6eb8846e2..ab0285b93d2 100644--- a/src/metabase/llm/api/provider.clj+++ b/src/metabase/llm/api/provider.clj@@ -153,8 +153,13 @@A type that names its model in `:config` (Azure, whose deployments its listing endpoint does not return) makesthe call for the same reason, but the model it serves comes from the connection rather than from the empty list- that comes back."- [{:keys [type config]} config-override model]+ that comes back.++ `probe?` asks a type that can check more than its credentials to do so — vLLM exercises the tool calling and+ structured output the agent loop depends on against the model it will run on. Only [[verify-credentials!]] sets+ it: a probe generates, so it is far too slow for a plain listing. A probe reports back the model it exercised as+ `:probed-model` and anything it determined about the connection as `:learned-config`, both passed through here."+ [{:keys [type config]} config-override model probe?](let [fixed (llm.provider/fixed-models type)](if (llm.provider/managed-type? type){:models (vec fixed)}@@ -162,12 +167,14 @@configured-model (llm.provider/connection-model type config)model (or model configured-model (:id (first fixed)))](try- (let [listed (:models (metabot.self/list-models type (cond-> {:credentials config}- model (assoc :model model))))]- {:models (cond- configured-model [{:id configured-model :display_name (last (str/split configured-model #"/"))}]- fixed (vec fixed)- :else (vec listed))})+ (let [listed (metabot.self/list-models type (cond-> {:credentials config}+ model (assoc :model model)+ probe? (assoc :probe? true)))]+ (merge (select-keys listed [:probed-model :learned-config])+ {:models (cond+ configured-model [{:id configured-model :display_name (last (str/split configured-model #"/"))}]+ fixed (vec fixed)+ :else (vec (:models listed)))}))(catch clojure.lang.ExceptionInfo e(if (provider-client-error? e){:models [] :error (.getMessage e)}@@ -193,7 +200,7 @@(merge {:key conn-key :name conn-name :type type}(try(cache.wrapped/lookup-or-miss models-cache (models-cache-key conn)- (fn [_] (list-connection-models* conn nil nil)))+ (fn [_] (list-connection-models* conn nil nil false)))(catch Exception e(log/warn e "Failed to list models for LLM provider connection" {:connection conn-key}){:models [] :error (.getMessage e)}))))@@ -224,18 +231,36 @@(into {} (remove (fn [[_ v]] (str/blank? v))) config))(defn- verify-credentials!- "Confirm `config` can actually reach `conn`'s provider before anything is persisted, by listing its models.- Throws a 400 carrying the provider's own message when the credentials are rejected.+ "Confirm `config` can actually reach `conn`'s provider before anything is persisted, by listing its models — and,+ for a type that probes more than its credentials, by exercising the model it will run on. Throws a 400 carrying+ the provider's own message when the credentials are rejected.++ Returns `{:probed-model :learned-config}`: the model a probe exercised, for a type with no default model to fall+ back on, and whatever the probe determined about the connection, for the caller to store on it.The listing that verified the credentials is seeded into [[models-cache]] under the connection as it will bestored, so the model refetch the client fires right after saving is answered from it instead of round-trippingto the provider a second time."[conn config model](when-not (llm.provider/managed-type? (:type conn))- (let [{:keys [error] :as listed} (list-connection-models* conn config model)]+ (let [{:keys [error learned-config] :as listed} (list-connection-models* conn config model true)](when error(throw (ex-info error {:status-code 400 :api-error true})))- (swap! models-cache cache/miss (models-cache-key (assoc conn :config config)) listed))))+ (swap! models-cache cache/miss+ (models-cache-key (assoc conn :config (merge config learned-config)))+ (select-keys listed [:models]))+ (select-keys listed [:probed-model :learned-config]))))++(defn- selected-model+ "The model `llm-metabot-provider` names for `conn-key`, or nil when the selection points elsewhere.++ What an edited connection is verified against when the client names no model: for a type that checks more than+ its credentials, the model the connection is actually serving is a better probe target than whichever one the+ provider happens to list first."+ [conn-key]+ (let [model-ref (metabot.settings/llm-metabot-provider)]+ (when (= conn-key (llm.provider/model-ref->connection-key model-ref))+ (llm.provider/model-ref->model model-ref))))(defn- connection-model-ref"The `connection-key/model` reference that points Metabot at `conn`: the model the connection's own config names@@ -383,12 +408,15 @@:name (or (not-empty name) (str (:label provider-type))):config config}](llm.provider/validate-config! type config)- (verify-credentials! conn config model)- (let [had-usable-model? (metabot-has-a-usable-model?)]+ (let [{:keys [probed-model learned-config]} (verify-credentials! conn config model)+ conn (update conn :config merge learned-config)+ had-usable-model? (metabot-has-a-usable-model?)](llm.provider/set-connections! (conj (llm.provider/stored-connections) conn))(when-not had-usable-model?- (select-model-for-new-connection! conn model)))- (connection-response (assoc conn :source :db)))))+ ;; a type with no default model — vLLM, which serves whatever the operator loaded — starts on the model+ ;; the probe exercised, so connecting one leaves the instance working rather than model-less+ (select-model-for-new-connection! conn (or model probed-model)))+ (connection-response (assoc conn :source :db))))))(api.macros/defendpoint :put "/providers/:key":- connection-response-schema@@ -420,10 +448,12 @@;; what the connection will actually run on: the stored config with the environment layered back over iteffective (merge (:config merged) env-config)](llm.provider/validate-config! (:type merged) effective)- (verify-credentials! merged effective model)- (llm.provider/set-connections! (assoc stored idx merged))- (follow-edited-connection-model! (assoc merged :config effective) model)- (connection-response (assoc (merge live merged) :config effective))))+ (let [{:keys [learned-config]} (verify-credentials! merged effective (or model (selected-model conn-key)))+ merged (update merged :config merge learned-config)+ effective (merge effective learned-config)]+ (llm.provider/set-connections! (assoc stored idx merged))+ (follow-edited-connection-model! (assoc merged :config effective) model)+ (connection-response (assoc (merge live merged) :config effective)))))(api.macros/defendpoint :delete "/providers/:key" :- :nil"Delete a provider connection."diff --git a/src/metabase/llm/provider.clj b/src/metabase/llm/provider.cljindex 53780bb12fc..fc24ebc53f4 100644--- a/src/metabase/llm/provider.clj+++ b/src/metabase/llm/provider.clj@@ -265,6 +265,25 @@:type :password:advanced? true:help (deferred-tru "Only needed for temporary credentials.")}]}+ {:type "vllm"+ :label (deferred-tru "vLLM")+ ;; A vLLM server serves whatever the operator loaded it with, so there is no model to default to: the one a+ ;; new connection starts on comes from the catalog that connecting fetches (see+ ;; [[metabase.metabot.self.vllm/list-models]]).+ :default-model nil+ :fields [{:key :base-url+ :normalize strip-trailing-slashes+ :label (deferred-tru "API base URL")+ :type :text+ :required? true+ :placeholder "http://vllm.internal:8000/v1"+ :help (deferred-tru "Your server''s OpenAI-compatible API. It should end in /v1.")}+ {:key :api-key+ :label (deferred-tru "API key")+ :type :password+ ;; not required: a server started without --api-key takes no key, and a base URL on its own+ ;; is a complete configuration+ :help (deferred-tru "Only needed if you started your server with --api-key.")}]}{:type "metabase":label (deferred-tru "Metabase AI service"):managed? true@@ -487,7 +506,12 @@:settings {:access-key-id {:setting :llm-bedrock-access-key-id :credential? true}:secret-access-key {:setting :llm-bedrock-secret-access-key :credential? true}:session-token {:setting :llm-bedrock-session-token}- :region {:setting :llm-bedrock-region}}}})+ :region {:setting :llm-bedrock-region}}}+ "vllm" {:type "vllm"+ ;; the base URL is the credential here, unlike Azure's: a server started without --api-key takes+ ;; no key, so the URL alone brings a usable connection into existence+ :settings {:base-url {:setting :llm-vllm-api-base-url :credential? true}+ :api-key {:setting :llm-vllm-api-key}}}})(defn- env-supplied-fields"The `:config` fields the environment supplies for one [[single-provider-settings]] group:diff --git a/src/metabase/llm/settings.clj b/src/metabase/llm/settings.cljindex a323e6685b1..3ec6f6e5a7e 100644--- a/src/metabase/llm/settings.clj+++ b/src/metabase/llm/settings.clj@@ -387,6 +387,34 @@:setter (connection-field-setter :llm-azure-deployment-name):doc "Backed by the azure connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.")+;;; --------------------------------------------------- vLLM ----------------------------------------------------++(defsetting llm-vllm-api-base-url+ (deferred-tru "The base URL of your vLLM server''s OpenAI-compatible API, e.g. `http://vllm.internal:8000/v1`.")+ :encryption :no+ :visibility :settings-manager+ :export? false+ :getter (connection-field-getter :llm-vllm-api-base-url)+ :setter (connection-field-setter :llm-vllm-api-base-url)+ :doc "Backed by the vllm connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.")++(defsetting llm-vllm-api-key+ (deferred-tru "The API key for your vLLM server. Only needed when the server was started with `--api-key`.")+ :sensitive? true+ :visibility :settings-manager+ :export? false+ :getter (connection-field-getter :llm-vllm-api-key)+ :setter (connection-field-setter :llm-vllm-api-key)+ :doc "Backed by the vllm connection in the admin AI settings provider list: reads and writes go through the llm-providers connection list, and a value set by this environment variable shadows this one field of that connection.")++(defsetting llm-vllm-request-timeout-ms+ (deferred-tru "Socket timeout in milliseconds for requests to your vLLM server.")+ ;; Self-hosted TTFT is bounded by the operator's hardware; the shared 60s default is too short.+ :type :integer+ :default 300000+ :visibility :settings-manager+ :export? false)+;;; ---------------------------------------------- Provider connections ------------------------------------------;;; The per-provider credential settings above are read-only at runtime: they configure a connection only when setdiff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 885b6b74db6..c0d35685e65 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -24,6 +24,7 @@[metabase.metabot.self.moonshot :as moonshot][metabase.metabot.self.openai :as openai][metabase.metabot.self.openrouter :as openrouter]+ [metabase.metabot.self.vllm :as vllm][metabase.metabot.self.zai :as zai][metabase.metabot.settings :as metabot.settings][metabase.metabot.usage :as usage]@@ -45,6 +46,7 @@"moonshot" moonshot/moonshot"openai" openai/openai"openrouter" openrouter/openrouter+ "vllm" vllm/vllm"zai" zai/zai(throw (ex-info (str "Unknown LLM provider: " provider){:provider provider}))))@@ -60,6 +62,7 @@"moonshot" moonshot/list-models"openai" openai/list-models"openrouter" openrouter/list-models+ "vllm" vllm/list-models"zai" zai/list-models(throw (ex-info (str "Unknown LLM provider: " provider){:provider provider}))))@@ -141,14 +144,21 @@error: `rethrow-api-error!` rethrows e.g. a `Read timed out` as an`ExceptionInfo` (with `:error-code :provider-request-failed` and no `:status`)whose *cause* is the original `SocketTimeoutException`. Inspecting only the- top-level exception would miss it and we'd never retry a transient timeout."+ top-level exception would miss it and we'd never retry a transient timeout.++ An adapter opts a specific failure out of that default by tagging its ex-data+ `:retryable? false` (top-level only), which wins over both checks. Note that+ omitting `:status` does not — the cause walk still matches."[^Exception e]- (boolean- (or (retryable-status? (:status (ex-data e)))- ;; Connection errors (e.g. under load, connection refused/reset), possibly- ;; wrapped one or more levels deep by a provider adapter. Bounded to 10- ;; levels to guard against a cyclic getCause chain.- (some connection-error? (take 10 (take-while some? (iterate #(some-> ^Throwable % .getCause) e)))))))+ (let [data (ex-data e)]+ (if (false? (:retryable? data))+ false+ (boolean+ (or (retryable-status? (:status data))+ ;; Connection errors (e.g. under load, connection refused/reset), possibly+ ;; wrapped one or more levels deep by a provider adapter. Bounded to 10+ ;; levels to guard against a cyclic getCause chain.+ (some connection-error? (take 10 (take-while some? (iterate #(some-> ^Throwable % .getCause) e)))))))))(defn- parse-retry-after-header"Extract retry-after seconds from response headers in ex-data, if present and ≤ 60s.diff --git a/src/metabase/metabot/self/openai/chat_completions.clj b/src/metabase/metabot/self/openai/chat_completions.cljindex b57b6ebf0ad..30af804eb42 100644--- a/src/metabase/metabot/self/openai/chat_completions.clj+++ b/src/metabase/metabot/self/openai/chat_completions.clj@@ -111,6 +111,15 @@"function_call" "tool-calls""content_filter" "content-filter"})+(defn- delta-reasoning+ "Reasoning text carried by a Chat Completions delta or message, under either spelling. vLLM 0.26+ emits `reasoning` and treats `reasoning_content` as its deprecated name; older builds, Z.AI, and+ other OpenAI-compatible servers still emit the latter, and a self-hosted server's version is the+ customer's choice."+ [m]+ (or (not-empty (:reasoning m))+ (not-empty (:reasoning_content m))))+(defn chat-completions->aisdk-chunks-xf"Translates Chat Completions streaming chunks into AI SDK v5 protocol chunks.@@ -138,13 +147,21 @@would lose their arguments, since neither the start branch (needs `:name`) northe argument-delta branch (needs no `:id`) would fire.- Takes the dialect's `finish_reason` table, defaulting to OpenAI's [[stop-reasons]]."+ Takes the dialect's `finish_reason` table, defaulting to OpenAI's [[stop-reasons]].++ `opts` may carry `:forward-reasoning?`, which additionally translates reasoning+ deltas (see [[delta-reasoning]]) into :reasoning-start / :reasoning-delta /+ :reasoning-end. Opt-in, because whether a provider's reasoning renders at all is+ a separate question (see `metabot.settings/llm-metabot-supports-reasoning?`) and+ chunks nothing consumes only add stream volume."([]- (chat-completions->aisdk-chunks-xf stop-reasons))+ (chat-completions->aisdk-chunks-xf stop-reasons nil))([stop-reasons]+ (chat-completions->aisdk-chunks-xf stop-reasons nil))+ ([stop-reasons {:keys [forward-reasoning?]}](fn [rf]- (let [current-type (volatile! nil) ;; :text | :function_call | nil- current-id (volatile! nil) ;; active chunk id (text-id or tool call_id)+ (let [current-type (volatile! nil) ;; :text | :reasoning | :function_call | nil+ current-id (volatile! nil) ;; active chunk id (text-id, reasoning-id, or tool call_id)message-id (volatile! nil)model-name (volatile! nil)payload (volatile! {}) ;; carried across start/delta/end, same as openai.clj@@ -152,6 +169,7 @@close! (fn [result](u/prog1 (rf result (merge {:type (case @current-type:text :text-end+ :reasoning :reasoning-end:function_call :tool-input-available)}@payload))(vreset! current-type nil)@@ -172,9 +190,11 @@;; Empty-string content (common between tool calls) is ignored;; to avoid spurious text blocks that would close open tools.chunk-type (cond- (not-empty (:content delta)) :text- (some? tool-call) :function_call- :else nil)+ (not-empty (:content delta)) :text+ (and forward-reasoning?+ (delta-reasoning delta)) :reasoning+ (some? tool-call) :function_call+ :else nil);; For new tool calls, the id comes from the chunk; for deltas;; on the same tool, we keep current-id.chunk-id (or (:id tool-call) @current-id (core/mkid))]@@ -204,6 +224,18 @@(some? (:content delta))) (rf {:type :text-delta:id @current-id:delta (:content delta)})+ ;; Start a new reasoning block+ (and (= chunk-type :reasoning)+ (not= @current-type :reasoning)) (-> (u/prog1+ (let [rid (core/mkid)]+ (vreset! current-type :reasoning)+ (vreset! current-id rid)+ (vreset! payload {:id rid})))+ (rf (merge {:type :reasoning-start} @payload)))+ ;; Reasoning delta+ (= chunk-type :reasoning) (rf {:type :reasoning-delta+ :id @current-id+ :delta (delta-reasoning delta)});; Start a new tool call block(and (= chunk-type :function_call)(:id tool-call)@@ -279,11 +311,15 @@no `:status`: this isn't a credentials problem, and `metabase.metabot.api`'s `provider-client-error?`renders any 4xx under the admin API-key field, which would attach the wrong message to the wrong input.- A well-formed but empty `data` is a legitimate response — an account with no accessible models — and passes."- [provider-name res]- (let [data (get-in res [:body :data])]- (when-not (sequential? data)- (throw (ex-info (tru "{0} returned an unexpected model list response" provider-name)- {:api-error true- :error-code :malformed-model-catalog})))- data))+ A well-formed but empty `data` is a legitimate response — an account with no accessible models — and passes.++ `:detail` is a sentence appended to the message, for a provider that has something more specific to say."+ ([provider-name res] (models-catalog provider-name res nil))+ ([provider-name res {:keys [detail]}]+ (let [data (get-in res [:body :data])]+ (when-not (sequential? data)+ (throw (ex-info (cond-> (tru "{0} returned an unexpected model list response" provider-name)+ detail (str ". " detail))+ {:api-error true+ :error-code :malformed-model-catalog})))+ data)))diff --git a/src/metabase/metabot/self/openrouter.clj b/src/metabase/metabot/self/openrouter.cljindex 6371fede208..768d7cbf42e 100644--- a/src/metabase/metabot/self/openrouter.clj+++ b/src/metabase/metabot/self/openrouter.clj@@ -135,6 +135,28 @@[model](str/starts-with? (str model) "anthropic/"))+(defn- anthropic-current-gen?+ "Current-generation Claude on OpenRouter: Fable, Opus >= 4.7, Sonnet >= 5."+ [model]+ (or (str/starts-with? model "anthropic/claude-fable")+ (when-let [[_ family major minor] (re-find #"^anthropic/claude-(opus|sonnet)-(\d+)(?:\.(\d+))?" model)]+ (let [major (parse-long major)+ minor (or (some-> minor parse-long) 0)]+ (case family+ "opus" (or (> major 4) (and (= major 4) (>= minor 7)))+ "sonnet" (>= major 5))))))++(defn- model-supports-temperature?+ "Whether an OpenRouter model id accepts an explicit `temperature`. Two families reject it: OpenAI's+ GPT-5 and o-series, and current-generation Claude. Neither `openai.clj`'s nor `claude.clj`'s+ predicate can be reused — both are keyed to their own provider's id shape, and would silently+ return true for OpenRouter's `vendor/model` ids with dotted versions."+ [model]+ (let [model (str model)]+ (not (or (str/starts-with? model "openai/gpt-5")+ (re-find #"^openai/o\d" model)+ (anthropic-current-gen? model)))))+(def ^:private required-tool-choice-unsupported-models"Models that don't support `:tool_choice \"required\"`"#{"qwen/qwen3.8-max"})@@ -158,13 +180,20 @@unlike claude.clj we don't put a separate breakpoint there.Other models (OpenAI) keep the generic plain string system message: OpenAI prompt caching is automatic server-side- and takes no request markup."+ and takes no request markup.++ `:temperature` is dropped for models that reject it (see [[model-supports-temperature?]]). Gating it in the shared+ builder instead would apply these OpenRouter-specific rules to every Chat Completions adapter, including vLLM,+ whose model names are customer-chosen free text."[{:keys [model system] :as opts:or {model "anthropic/claude-haiku-4.5"}} :- core/LLMRequestOpts](cond-> (chat-completions/request-body (assoc opts :model model))(and system (anthropic-model? model))(update-in [:messages 0 :content] claude/system->cached-content-blocks)+ (not (model-supports-temperature? model))+ (dissoc :temperature)+(not (supports-required-tool-choice? model))required-tool-choice->auto))diff --git a/src/metabase/metabot/self/vllm.clj b/src/metabase/metabot/self/vllm.cljnew file mode 100644index 00000000000..c92f00a3113--- /dev/null+++ b/src/metabase/metabot/self/vllm.clj@@ -0,0 +1,524 @@+(ns metabase.metabot.self.vllm+ "vLLM / Chat Completions adapter for self-hosted, OpenAI-compatible inference servers.++ The base URL is required and the API key is optional, and there is no model whitelist — the+ operator serves whatever they loaded, so [[preflight!]] exercises the agent-loop contract at+ configuration time instead.++ https://docs.vllm.ai/en/latest/serving/openai_compatible_server.html"+ (:require+ [clojure.string :as str]+ [metabase.llm.settings :as llm]+ [metabase.metabot.self.core :as core]+ [metabase.metabot.self.debug :as debug]+ [metabase.metabot.self.openai.chat-completions :as chat-completions]+ [metabase.util :as u]+ [metabase.util.i18n :refer [tru]]+ [metabase.util.json :as json]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.o11y :refer [with-span]])+ (:import+ (java.io IOException)+ (java.net SocketTimeoutException)+ (java.util.concurrent ExecutionException)))++(set! *warn-on-reflection* true)++(def ^:private min-context-length+ "Smallest `max_model_len` [[preflight!]] will accept. A product floor, not a measurement."+ 16384)++(defn- ai-proxy-unsupported-ex []+ (ex-info (tru "AI proxy is not supported for vLLM")+ {:api-error true+ :error-code :proxy-unsupported}))++(defn- missing-base-url-ex []+ (ex-info (tru "No vLLM base URL is set")+ {:api-error true+ :error-code :base-url-missing}))++(defn- missing-model-ex []+ (ex-info (tru "No vLLM model is set")+ {:api-error true+ :error-code :model-missing}))++(defn- vllm-error-msg+ "Canonical, status-specific vLLM error message."+ [res]+ (let [status (long (:status res 0))]+ (case status+ 400 (tru "vLLM rejected the request — usually an unsupported schema, or a model that cannot compile the tool grammar")+ 401 (tru "vLLM API key expired or invalid — check the key your server was started with via --api-key")+ 404 (tru "vLLM API endpoint was not found — the base URL should end in /v1")+ 429 (tru "The vLLM server''s request queue is full — reduce concurrent load, or restart it with a larger --max-num-seqs")+ 500 (tru "vLLM returned an internal server error")+ (tru "vLLM API error (HTTP {0})" status))))++(def reasoning-config-key+ "The connection `:config` key [[preflight!]]'s reasoning observation is recorded under. It is not an+ admin-entered field: whether a served model reasons depends on the operator's `--reasoning-parser`+ as much as on the model, so only the probe can answer it."+ :model-reasoning)++(defn reasoning-connection?+ "Whether the connection carrying `credentials` was observed streaming its reasoning. A hand-written+ `llm-providers` can hold a JSON boolean where the API stores the string it round-trips."+ [credentials]+ (let [recorded (get credentials reasoning-config-key)]+ (or (true? recorded) (= "true" recorded))))++(defn- vllm-auth+ "Auth map for a vLLM request, built from the connection's credentials alone. The map is never nil, so+ `core/resolve-auth` cannot reach its `missing-api-key-ex` branch — a keyless server is a complete+ configuration."+ [credentials ai-proxy?]+ (when ai-proxy?+ (throw (ai-proxy-unsupported-ex)))+ (let [base-url (not-empty (:base-url credentials))+ api-key (not-empty (:api-key credentials))]+ (when-not base-url+ (throw (missing-base-url-ex)))+ (core/resolve-auth "vllm" "vLLM"+ (cond-> {:url base-url}+ api-key (assoc :headers {"Authorization" (str "Bearer " api-key)}))+ ai-proxy?)))++(defn- inference-timeouts+ "Timeouts for a generation request."+ []+ {:socket-timeout (llm/llm-vllm-request-timeout-ms)+ :connection-timeout (llm/llm-connection-timeout-ms)})++(defn- control-timeouts+ "Timeouts for the non-generating `/models` call, on the shared (much shorter) request budget."+ []+ {:socket-timeout (llm/llm-request-timeout-ms)+ :connection-timeout (llm/llm-connection-timeout-ms)})++(def ^:private probe-timeout-ceiling-ms+ "Upper bound on a single preflight probe, which blocks the admin behind a spinner."+ 120000)++(defn- probe-timeouts+ "Timeouts for a preflight probe. An operator who lowers `llm-vllm-request-timeout-ms` below the+ ceiling gets their own value."+ []+ {:socket-timeout (min (llm/llm-vllm-request-timeout-ms) probe-timeout-ceiling-ms)+ :connection-timeout (llm/llm-connection-timeout-ms)})++;;; ----------------------------------------------- Transport errors ---------------------------------------------++(defn- unreachable-ex+ "The vLLM error for a non-timeout transport failure. `extra` is the caller's own ex-data tags."+ [^IOException e base-url extra]+ (ex-info (tru "Could not reach the vLLM server at {0}. Check that it is running and that the base URL is correct."+ (str base-url))+ (merge {:api-error true :error-code :vllm-unreachable} extra)+ e))++(defn- list-models-io-ex+ "The vLLM error for a transport failure while fetching the model catalog — the request behind the+ admin Connect button. Tagged `:status-code 400` so a mistyped base URL surfaces the message rather+ than the 500 `core/rethrow-api-error!`'s untagged no-response branch would produce."+ [^IOException e base-url]+ (if (instance? SocketTimeoutException e)+ (ex-info (tru "The vLLM server at {0} did not respond within {1}ms. Check that it is running and not overloaded."+ (str base-url) (str (llm/llm-request-timeout-ms)))+ {:api-error true+ :status-code 400+ :error-code :vllm-timeout}+ e)+ (unreachable-ex e base-url {:status-code 400})))++;;; ------------------------------------------------ Model listing -----------------------------------------------++(defn- list-all-models+ "Fetch the served model catalog, which doubles as the credential round-trip behind the admin+ Connect button.++ A 2xx whose body is not a recognizable catalog fails closed via+ [[chat-completions/models-catalog]], naming the base URL — the likeliest cause for the one provider+ whose base URL the admin types."+ [auth]+ (try+ (let [res (core/request auth (merge {:method :get+ :url "/models"+ :as :json+ :headers {"Content-Type" "application/json"}}+ (control-timeouts)))]+ ;; The URL off `auth`, not the setting: a connect verifies request credentials before saving them.+ (chat-completions/models-catalog+ "vLLM" res+ {:detail (tru "Check that {0} is a vLLM server''s OpenAI-compatible API — the base URL should end in /v1."+ (str (:url auth)))}))+ ;; Ordered ahead of the generic catch, which a non-2xx still reaches as an `ExceptionInfo`.+ ;; `:as :json` also lands a 2xx whose body is not JSON here, via Jackson's `JsonParseException`.+ (catch IOException e+ (throw (list-models-io-ex e (:url auth))))+ (catch Exception e+ (core/rethrow-api-error! "vllm" vllm-error-msg e))))++;;; -------------------------------------------------- Preflight -------------------------------------------------++(def ^:private probe-tool+ {:type "function"+ :function {:name "record_table_name"+ :description "Record the name of the table the user mentioned."+ :parameters {:type "object"+ :properties {:table_name {:type "string"+ :description "The table name the user mentioned."}}+ :required ["table_name"]+ :additionalProperties false}}})++(def ^:private probe-messages+ [{:role "user" :content "Record the table name: orders"}])++(def ^:private probe-max-tokens+ "Generation ceiling for a preflight probe. High enough to clear a reasoning model's thinking, which+ is billed against it: a probe that stops at `length` before the tool call looks identical to a+ server that will not call tools at all."+ 2048)++(def ^:private forced-tool-call-token-floor+ "Smallest `max_tokens` a forced tool call is given, regardless of what the caller asked for — below+ it a reasoning model spends the budget thinking and emits no tool call. Equal to+ [[probe-max-tokens]], which [[preflight!]] already proves the served model can clear."+ probe-max-tokens)++(def ^:private reasoning-model-token-floor+ "Smallest `max_tokens` any request gets once [[preflight!]] has observed the served model reasoning.+ Chat Completions bills thinking, answer, and tool call against one budget. Mirrors+ `claude-request-body`."+ 16384)++(def ^:private default-temperature+ "Sampling temperature for a caller that supplies none. vLLM's own default is 1.0, which is wrong for+ the tool-calling and SQL-generation work the agent loop does. The hosted providers pick a sane+ default server-side; a self-hosted server does not, so the adapter supplies one."+ 0.3)++(defn- preflight-ex+ "A preflight failure, tagged so `metabase.metabot.api` surfaces the message verbatim, not as a 500."+ [msg]+ (ex-info msg {:api-error true+ :status-code 400+ :error-code :vllm-preflight-failed}))++(defn- probe-chat!+ "Run one non-streaming Chat Completions turn against `model` and return the first choice. The+ `finish_reason` is part of the return value because a generation truncated at+ [[probe-max-tokens]] and a server that will not call tools both produce empty `tool_calls`."+ [auth model tool-choice]+ (let [res (core/request auth (merge {:method :post+ :url "/chat/completions"+ :as :json+ :headers {"Content-Type" "application/json"}+ :body (json/encode {:model model+ :messages probe-messages+ :tools [probe-tool]+ :tool_choice tool-choice+ :temperature 0+ :max_tokens probe-max-tokens})}+ (probe-timeouts)))]+ (get-in res [:body :choices 0])))++(defn- check-context-budget!+ [{:keys [id max_model_len]}]+ (when (and max_model_len (< (long max_model_len) min-context-length))+ (throw (preflight-ex+ (tru "{0} is served with a {1} token context window, which is too small for Metabot — it needs at least {2}. Restart vLLM with a larger --max-model-len."+ (str id) (str max_model_len) (str min-context-length))))))++(defn- check-tool-calling!+ "Check that the server was started with `--enable-auto-tool-choice` and a `--tool-call-parser` whose+ sentinels match this model. A wrong-but-valid parser name errors at no layer — the call stays in+ `content` as prose and Metabot chats without ever acting.++ Returns whether the model emitted reasoning, the only signal anywhere that it is a reasoning model."+ [auth model]+ (let [{:keys [message finish_reason]} (probe-chat! auth model "auto")+ content (str (:content message))+ ;; `reasoning` since vLLM 0.26; `reasoning_content` is the deprecated spelling older builds+ ;; and other OpenAI-compatible servers still use.+ reasoning (str (or (:reasoning message) (:reasoning_content message)))+ tool-calls (:tool_calls message)+ truncated? (= "length" finish_reason)]+ (cond+ (str/includes? content "<think>")+ (throw (preflight-ex+ (tru "{0} streamed its reasoning as chat text. Restart vLLM with --reasoning-parser so thinking doesn''t appear inside Metabot''s answers."+ (str model))))++ (seq tool-calls)+ (let [arguments (get-in (first tool-calls) [:function :arguments])]+ (when-not (try+ (map? (json/decode+kw (str arguments)))+ (catch Exception _ false))+ (throw (preflight-ex+ (if truncated?+ (tru "{0} reached the {1} token connection-test ceiling partway through a tool call. A model that generates this much before calling a tool is too slow to drive Metabot."+ (str model) (str probe-max-tokens))+ (tru "The vLLM server returned a tool call whose arguments are not valid JSON. The --tool-call-parser most likely does not match {0}''s output format."+ (str model))))))+ (not (str/blank? reasoning)))++ (and truncated? (not (str/blank? reasoning)))+ (throw (preflight-ex+ (tru "{0} spent the entire {1} token connection-test budget reasoning without calling a tool. A model that thinks this long about a trivial prompt is too slow to drive Metabot."+ (str model) (str probe-max-tokens))))++ ;; A tool call cut off at the ceiling reaches here, not the `(seq tool-calls)` branch above: the+ ;; parsers extract from complete output, so a call missing its closing sentinel yields no+ ;; `tool_calls` at all and leaves the raw text in `content`. That is indistinguishable from+ ;; prose except by `finish_reason`, and naming the flags — which are working, or the sentinel+ ;; would not be there — sends the admin to fix something that is not broken. A verbose model+ ;; whose flags really are missing lands here too, and gets the flags message on the retry after+ ;; raising the ceiling; truncation is the problem to fix first either way.+ truncated?+ (throw (preflight-ex+ (tru "{0} reached the {1} token connection-test ceiling before completing a tool call. A model that generates this much before calling a tool is too slow to drive Metabot."+ (str model) (str probe-max-tokens))))++ :else+ (throw (preflight-ex+ (tru "The vLLM server answered with text instead of calling a tool. Restart it with --enable-auto-tool-choice and a --tool-call-parser matching {0}''s output format."+ (str model)))))))++(defn- check-structured-output!+ "Check that guided decoding works. A different failure from [[check-tool-calling!]]: a model whose+ grammar the server cannot compile chats fine but breaks titling and the whole `sql` profile."+ [auth model]+ (let [{:keys [message finish_reason]} (probe-chat! auth model "required")]+ (when (empty? (:tool_calls message))+ (throw (preflight-ex+ (if (= "length" finish_reason)+ (tru "{0} reached the {1} token connection-test ceiling without producing a forced tool call. Metabot needs structured output support for conversation titles and SQL generation."+ (str model) (str probe-max-tokens))+ (tru "The vLLM server did not honor a forced tool call. Metabot needs structured output support for conversation titles and SQL generation.")))))))++(defn- no-models-ex []+ (preflight-ex (tru "The vLLM server is reachable but is not serving any models.")))++(defn- connect-candidates+ "Catalog entries eligible to be adopted when the connect path picks a model for the admin.++ Skips LoRA adapters, whose `parent` names the base model they adapt. Nothing in the catalog+ distinguishes an embedding or reranker deployment, so a multi-model server can still be adopted+ from wrongly — the admin re-picks from the dropdown."+ [entries]+ (or (seq (remove :parent entries)) entries))++(defn- probe-target+ "The catalog entry [[preflight!]] will probe.++ A requested model is the only acceptable target: falling back to another served model would pass+ every check and then persist a provider string naming a model the server does not have. The+ fallback is for the connect path, which supplies no model because the served name is knowable+ only from this catalog."+ [entries requested-model]+ (if requested-model+ (or (u/seek #(= requested-model (:id %)) entries)+ (throw (if (seq entries)+ (preflight-ex (tru "The vLLM server is not serving {0}. It is serving: {1}."+ (str requested-model) (str/join ", " (map :id entries))))+ (no-models-ex))))+ (or (first (connect-candidates entries))+ (throw (no-models-ex)))))++(defn- await-probe!+ "Deref a probe future, unwrapping the `ExecutionException` so the preflight's own `ex-info` reaches+ the caller with its `:api-error` tag intact."+ [fut]+ (try+ @fut+ (catch ExecutionException e+ (throw (or (ex-cause e) e)))))++(defn- run-probes!+ "Run both contract probes against `model` and return whether it streamed reasoning.++ They run concurrently; deref order fixes the verdict, tool calling being the more actionable+ diagnosis when a server fails both. The loser is cancelled rather than left generating against the+ operator's server long after anyone is listening."+ [auth model]+ (try+ (let [tool-calling (future (check-tool-calling! auth model))+ structured (future (check-structured-output! auth model))]+ (try+ (let [reasoning? (await-probe! tool-calling)]+ (await-probe! structured)+ (boolean reasoning?))+ (finally+ (future-cancel tool-calling)+ (future-cancel structured))))+ (catch SocketTimeoutException _+ (throw (preflight-ex+ (tru "The vLLM server did not answer the connection test within {0}ms. Check that it is not overloaded — a server this slow to answer a trivial prompt cannot drive Metabot."+ (str (:socket-timeout (probe-timeouts)))))))+ (catch Exception e+ (core/rethrow-api-error! "vllm" vllm-error-msg e))))++(defn- preflight!+ "Exercise the contract the agent loop depends on, against the model that will actually be used, and+ return `{:model id :reasoning? bool}`. The connect path must adopt exactly this model rather than+ re-deriving it from the listing, which agrees only while nothing reorders the catalog.++ `:reasoning?` reports whether the probed model streamed reasoning. Only the probe can answer that,+ and the answer drives which renderer the frontend picks, so the connection records it (see+ [[reasoning-config-key]])."+ [auth entries requested-model]+ (let [entry (probe-target entries requested-model)+ model (:id entry)]+ (check-context-budget! entry)+ {:model model+ :reasoning? (run-probes! auth model)}))++(defn list-models+ "List the models the connection's vLLM server is serving. Pass-through: there is nothing to+ whitelist, and `display_name` falls back to the served id.++ `:probe?` additionally runs [[preflight!]], and reports the model it exercised as `:probed-model`+ for the connect path to adopt, along with the `:learned-config` the probe determined. Reserved for+ the connect and edit paths — a tool-call probe on every model listing would stall the admin picker+ behind a full prefill."+ ([] (list-models {}))+ ([{:keys [credentials ai-proxy? model probe?]}]+ (let [auth (vllm-auth credentials ai-proxy?)+ entries (list-all-models auth)+ probed (when probe?+ (preflight! auth entries model))]+ (cond-> {:models (mapv (fn [{:keys [id] :as entry}]+ {:id id :display_name (or (:name entry) id)})+ entries)}+ probed (assoc :probed-model (:model probed)+ :learned-config {reasoning-config-key (str (:reasoning? probed))})))))++;;; --------------------------------------------------- Requests -------------------------------------------------++(mu/defn vllm-request-body+ "Build the Chat Completions request body for an LLM request.++ Matches what [[chat-completions/request-body]] emits, except that `max_tokens` is always sent —+ without a ceiling vLLM falls back to the remaining context window, so one looping small model+ consumes the whole budget in a single call — and is raised to+ [[forced-tool-call-token-floor]] or [[reasoning-model-token-floor]] where either applies, and+ `temperature` falls back to [[default-temperature]]. All three stay adapter-local rather than+ moving into the shared builder, which would also change Z.AI, Mistral, and OpenRouter."+ [{:keys [max-tokens temperature schema tool_choice credentials] :as opts} :- core/LLMRequestOpts]+ (let [forced? (or (some? schema) (= "required" (some-> tool_choice name)))]+ (assoc (chat-completions/request-body (cond-> opts+ (nil? temperature) (assoc :temperature default-temperature)))+ :max_tokens (cond-> (or max-tokens (llm/llm-max-tokens))+ forced? (max forced-tool-call-token-floor)+ (reasoning-connection? credentials) (max reasoning-model-token-floor)))))++(defn- stream-io-ex+ "The vLLM error for a transport failure while *consuming* a response stream. Tagged+ `:retryable? false`: on a self-hosted server a stalled or severed response means \"too slow\" or+ \"it died\", not \"transient\", and a retry replays a full cold prefill at up to+ `llm-vllm-request-timeout-ms` (300s) apiece. The tag is required — `retryable-error?` walks the+ cause chain and would otherwise match the `IOException` below."+ [^IOException e timeout-ms]+ (if (instance? SocketTimeoutException e)+ (ex-info (tru "The vLLM server stopped responding after {0}ms. Raise the vLLM request timeout, or serve a faster model."+ (str timeout-ms))+ {:api-error true+ :error-code :vllm-timeout+ :retryable? false}+ e)+ (ex-info (tru "The connection to the vLLM server was interrupted before the response finished.")+ {:api-error true+ :error-code :vllm-stream-interrupted+ :retryable? false}+ e)))++(defn- request-io-ex+ "The vLLM error for a transport failure while *establishing* a request. `core/rethrow-api-error!`+ would render these as \"vllm API request failed: Read timed out\", naming neither the server's+ slowness nor the setting that governs it.++ Tagged `:retryable? false` for the same reason as [[stream-io-ex]], and more importantly: nothing+ has been emitted yet, so `call-llm`'s own \"nothing emitted\" predicate would not stop a replay."+ [^IOException e base-url timeout-ms]+ (if (instance? SocketTimeoutException e)+ (ex-info (tru "The vLLM server did not respond within {0}ms. Check that it is not overloaded, or raise the vLLM request timeout."+ (str timeout-ms))+ {:api-error true+ :error-code :vllm-timeout+ :retryable? false}+ e)+ (unreachable-ex e base-url {:retryable? false})))++(defn- io-guarded+ "Wrap a stream reducible so a transport failure while consuming it surfaces as [[stream-io-ex]]+ rather than a raw `IOException`. The adapter's own `try` covers only establishing the request.++ Goes inside `core/reducible-with-api-errors`, never outside: [[stream-io-ex]] tags `:api-error+ true`, which `core/rethrow-api-error!` rethrows unchanged, so this translation wins for IO."+ [reducible timeout-ms]+ (reify clojure.lang.IReduceInit+ (reduce [_ rf init]+ (try+ (.reduce ^clojure.lang.IReduceInit reducible rf init)+ (catch IOException e+ (throw (stream-io-ex e timeout-ms)))))))++(mu/defn vllm-raw+ "Perform a streaming request to a vLLM server's Chat Completions API.+ Opts map takes `:credentials` (`{:base-url ... :api-key ...}`) from the connection serving this+ request, and throws without a base URL.+ `:ai-proxy?` is not supported for vLLM and throws when true."+ [{:keys [model tools credentials ai-proxy?] :as opts} :- core/LLMRequestOpts]+ (when ai-proxy?+ (throw (ai-proxy-unsupported-ex)))+ (when (str/blank? model)+ (throw (missing-model-ex)))+ (let [req (vllm-request-body opts)+ timeout-ms (llm/llm-vllm-request-timeout-ms)]+ (log/debug "vLLM request" {:model model :msg-count (count (:messages req)) :tools (count (or tools []))})+ (with-span :info {:name :metabot.vllm/request+ :model model+ :msg-count (count (:messages req))+ :tool-count (count (or tools []))}+ (try+ (let [auth (vllm-auth credentials ai-proxy?)+ response (core/request auth+ (merge {:method :post+ :url "/chat/completions"+ :as :stream+ :headers {"Content-Type" "application/json"}+ :body (json/encode req)}+ (inference-timeouts)))]+ (-> (core/sse-reducible (:body response))+ (debug/capture-stream {:provider "vllm"+ :model model+ :url "/chat/completions"+ :request req})+ (io-guarded timeout-ms)+ (core/reducible-with-api-errors "vllm" vllm-error-msg)))+ ;; Ordered: clj-http raises an `IOException` only when there is no response at all, so this+ ;; cannot swallow one `vllm-error-msg` would have translated.+ (catch IOException e+ (throw (request-io-ex e (:base-url credentials) timeout-ms)))+ (catch Exception e+ (core/rethrow-api-error! "vllm" vllm-error-msg e))))))++(defn vllm->aisdk-chunks-xf+ "Translates vLLM Chat Completions streaming chunks into AI SDK v5 protocol chunks.++ A model started with `--reasoning-parser` routes thinking to `delta.reasoning` and its answer back+ to `delta.content`; both are forwarded. The branch is self-gating — the field is present only when+ the served model reasons — so nothing here needs to know which model is loaded.++ vLLM adds no `finish_reason` beyond OpenAI's, so it takes the base stop-reason table."+ []+ (chat-completions/chat-completions->aisdk-chunks-xf chat-completions/stop-reasons+ {:forward-reasoning? true}))++(defn vllm+ "Call a vLLM server's Chat Completions API, return AISDK stream."+ [& args]+ (let [raw (apply vllm-raw args)]+ (eduction (vllm->aisdk-chunks-xf) raw)))diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.cljindex 94483b13ffc..d81cf99c861 100644--- a/src/metabase/metabot/settings.clj+++ b/src/metabase/metabot/settings.clj@@ -5,6 +5,7 @@[metabase.llm.settings :as llm.settings][metabase.metabot.self.claude :as claude][metabase.metabot.self.openai :as openai]+ [metabase.metabot.self.vllm :as vllm][metabase.settings.core :as setting :refer [defsetting]][metabase.util.i18n :refer [deferred-tru tru]][metabase.util.log :as log]))@@ -221,12 +222,17 @@:doc false)(defn- llm-provider-streams-reasoning?- "Whether a model reference names a model that streams its reasoning back to us."+ "Whether a model reference names a model that streams its reasoning back to us.++ Anthropic and OpenAI answer from the model name, because thinking is requested in the request body. vLLM answers+ from what its connect-time probe observed and recorded on the connection — the flag depends on the operator's+ `--reasoning-parser` as well as on the model, so the name cannot settle it."[model-ref]- (let [{:keys [type model]} (llm.provider/resolve-model-ref model-ref)]+ (let [{:keys [type model credentials]} (llm.provider/resolve-model-ref model-ref)](case type"anthropic" (claude/reasoning-model? model)"openai" (openai/reasoning-model? model)+ "vllm" (vllm/reasoning-connection? credentials)false)))(defsetting llm-metabot-supports-reasoning?diff --git a/test/metabase/llm/api/provider_test.clj b/test/metabase/llm/api/provider_test.cljindex 5463dd65a84..4ef2509c90b 100644--- a/test/metabase/llm/api/provider_test.clj+++ b/test/metabase/llm/api/provider_test.clj@@ -1,5 +1,6 @@(ns metabase.llm.api.provider-test(:require+ [clj-http.client :as http][clojure.test :refer [deftest is testing use-fixtures]][metabase.llm.api.provider :as llm.api.provider][metabase.llm.provider :as llm.provider]@@ -30,9 +31,10 @@(deftest provider-types-test(testing "every provider type is listed with the credential fields a connection needs"(let [types (mt/user-http-request :crowberto :get 200 "llm/provider-types")]- (is (= #{"anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "metabase"}+ (is (= #{"anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "vllm"+ "metabase"}(set (map :type types))))- (is (= ["anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock"]+ (is (= ["anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "vllm"](remove #{"metabase"} (map :type types)))"the bring-your-own-key providers keep their registry order")(is (=? {:type "anthropic"@@ -234,6 +236,97 @@{:type "openai" :config {:api-key "sk-valid"}})(is (= "anthropic/claude-opus-4-8" (metabot.settings/llm-metabot-provider))))))))+(deftest create-vllm-connection-adopts-the-model-its-probe-exercised-test+ (testing (str "A vLLM server serves whatever the operator loaded, so there is no default model to select: "+ "connecting adopts the model the connect-time probe actually ran the agent-loop contract "+ "against, and records what that probe could only learn by running.")+ (let [opts (atom nil)]+ (mt/with-dynamic-fn-redefs [metabot.self/list-models+ (fn [_provider o]+ (reset! opts o)+ {:models [{:id "vllm-test" :display_name "vllm-test"}+ {:id "other" :display_name "other"}]+ :probed-model "vllm-test"+ :learned-config {:model-reasoning "true"}})]+ (mt/with-temporary-setting-values [llm-providers []]+ (mt/with-temporary-raw-setting-values [llm-metabot-provider nil]+ (is (=? {:key "vllm"+ :type "vllm"+ :usable true+ :config {:base-url "http://vllm.internal:8000/v1"}}+ (mt/user-http-request :crowberto :post 200 "llm/providers"+ {:type "vllm"+ :config {:base-url "http://vllm.internal:8000/v1"}})))+ (testing "the connection is verified by a probe, not by a plain listing"+ (is (=? {:credentials {:base-url "http://vllm.internal:8000/v1"} :probe? true} @opts)))+ (is (= "vllm/vllm-test" (metabot.settings/llm-metabot-provider)))+ (testing "and what the probe learned is stored on the connection, where the request path reads it"+ (is (= {:base-url "http://vllm.internal:8000/v1"+ :model-reasoning "true"}+ (stored-config "vllm")))+ (is (true? (metabot.settings/llm-metabot-supports-reasoning?))))))))))++(deftest create-vllm-connection-is-rejected-when-the-probe-fails-test+ (testing "a server that cannot drive the agent loop is not saved, so nothing points Metabot at it"+ (mt/with-dynamic-fn-redefs [metabot.self/list-models+ (fn [& _]+ (throw (ex-info "The vLLM server answered with text instead of calling a tool."+ {:api-error true :status-code 400})))]+ (mt/with-temporary-setting-values [llm-providers []]+ (is (=? {:message "The vLLM server answered with text instead of calling a tool."}+ (mt/user-http-request :crowberto :post 400 "llm/providers"+ {:type "vllm"+ :config {:base-url "http://vllm.internal:8000/v1"}})))+ (is (= [] (llm.provider/connections)))))))++(deftest create-vllm-connection-rejects-an-unreachable-base-url-test+ (testing (str "vLLM is the one provider whose base URL the admin types, so a typo has to come back as the "+ "adapter's message on the form — not as the 500 an untagged transport failure would produce.")+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (java.net.ConnectException. "Connection refused")))]+ (mt/with-temporary-setting-values [llm-providers []]+ (is (= (str "Could not reach the vLLM server at http://vllm.internal:8000/v1. "+ "Check that it is running and that the base URL is correct.")+ (:message (mt/user-http-request :crowberto :post 400 "llm/providers"+ {:type "vllm"+ :config {:base-url "http://vllm.internal:8000/v1"}}))))+ (is (= [] (llm.provider/connections)))))))++(deftest models-listing-does-not-probe-test+ (testing "listing models is a page load; only a write may spend a generation on the operator's server"+ (let [opts (atom nil)]+ (mt/with-dynamic-fn-redefs [metabot.self/list-models+ (fn [_provider o]+ (reset! opts o)+ {:models [{:id "vllm-test" :display_name "vllm-test"}]})]+ (mt/with-temporary-setting-values [llm-providers [(connection "vllm" "vllm"+ {:base-url "http://vllm.internal:8000/v1"})]]+ (is (=? [{:key "vllm" :models [{:id "vllm-test"}]}]+ (mt/user-http-request :crowberto :get 200 "llm/models")))+ (is (not (:probe? @opts))))))))++(deftest update-verifies-against-the-model-the-connection-serves-test+ (testing (str "An edit that names no model is verified against the model this connection is actually serving "+ "Metabot, rather than whichever one the provider happens to list first.")+ (let [opts (atom nil)]+ (mt/with-dynamic-fn-redefs [metabot.self/list-models+ (fn [_provider o]+ (reset! opts o)+ {:models [{:id "served-a" :display_name "served-a"}]+ :probed-model "served-b"+ :learned-config {:model-reasoning "false"}})]+ (mt/with-temporary-setting-values [llm-providers [(connection "vllm" "vllm"+ {:base-url "http://old.internal:8000/v1"+ :model-reasoning "true"})]]+ (mt/with-temporary-raw-setting-values [llm-metabot-provider "vllm/served-b"]+ (mt/user-http-request :crowberto :put 200 "llm/providers/vllm"+ {:config {:base-url "http://vllm.internal:8000/v1"}})+ (is (=? {:model "served-b" :probe? true} @opts))+ (testing "and the probe's fresh verdict replaces the one the connection was carrying"+ (is (= {:base-url "http://vllm.internal:8000/v1"+ :model-reasoning "false"}+ (stored-config "vllm")))+ (is (false? (metabot.settings/llm-metabot-supports-reasoning?))))))))))+(deftest create-selects-a-model-composed-from-the-config-test(testing (str "Azure names its deployment in `:config` rather than listing models, and the form leaves a field it ""pre-filled with the registry default out of the payload, so the default has to be filled in before "diff --git a/test/metabase/llm/provider_test.clj b/test/metabase/llm/provider_test.cljindex 4e42cd7654a..0aaf60f8221 100644--- a/test/metabase/llm/provider_test.clj+++ b/test/metabase/llm/provider_test.clj@@ -218,6 +218,13 @@"connection without one still counts as complete")(is (true? (llm.provider/config-complete? "azure" {:api-key "azure-key":base-url "https://r.services.ai.azure.com/openai"})))))+ (testing "vLLM needs a base URL, and its API key is optional — a server started without --api-key takes none"+ (is (true? (llm.provider/config-complete? "vllm" {:base-url "http://vllm.internal:8000/v1"})))+ (is (true? (llm.provider/config-complete? "vllm" {:base-url "http://vllm.internal:8000/v1"+ :api-key "local-dev-key"})))+ (is (false? (llm.provider/config-complete? "vllm" {:api-key "local-dev-key"})))+ (is (false? (llm.provider/config-complete? "vllm" {:base-url " "})))+ (is (false? (llm.provider/config-complete? "vllm" nil))))(testing "bedrock needs both AWS keys, and neither the region nor the session token"(is (true? (llm.provider/config-complete? "bedrock" {:access-key-id "AKIAIOSFODNN7EXAMPLE":secret-access-key "test-secret"})))@@ -256,6 +263,17 @@:config {:api-key "sk-ant-db"}:source :db}](llm.provider/connections)))))+ (testing "stored and env-derived connections are merged, stored first"+ (mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic" {:api-key "sk-ant-db"})]]+ (mt/with-temp-env-var-value! [mb-llm-openai-api-key "sk-env"]+ (is (= [["anthropic" :db] ["openai" :env]]+ (map (juxt :key :source) (llm.provider/connections)))))))+ (testing "the whole stored list is read-only when it comes from the environment"+ (mt/with-temp-env-var-value! [mb-llm-providers "[{\"key\":\"anthropic\",\"type\":\"anthropic\",\"name\":\"Anthropic\",\"config\":{\"api-key\":\"sk-ant-env\"}}]"]+ (is (= [["anthropic" :env]]+ (map (juxt :key :source) (llm.provider/connections)))))))++(deftest connections-synthesized-from-the-environment-test(testing "the single-provider environment variables synthesize read-only connections keyed by provider type"(mt/with-temporary-setting-values [llm-providers []](mt/with-temp-env-var-value! [mb-llm-anthropic-api-key "sk-ant-env"@@ -272,11 +290,29 @@(mt/with-temporary-setting-values [llm-providers []](mt/with-temp-env-var-value! [mb-llm-anthropic-api-base-url "https://env.example.com"](is (= [] (llm.provider/connections))))))- (testing "stored and env-derived connections are merged, stored first"- (mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic" {:api-key "sk-ant-db"})]]- (mt/with-temp-env-var-value! [mb-llm-openai-api-key "sk-env"]- (is (= [["anthropic" :db] ["openai" :env]]- (map (juxt :key :source) (llm.provider/connections)))))))+ (testing "vLLM's base URL is its credential, so it alone synthesizes a connection"+ (mt/with-temporary-setting-values [llm-providers []]+ (mt/with-temp-env-var-value! [mb-llm-vllm-api-base-url "http://vllm.internal:8000/v1"]+ (is (= [{:key "vllm"+ :type "vllm"+ :name "vLLM"+ :source :env+ :env-vars #{"MB_LLM_VLLM_API_BASE_URL"}+ :env-fields #{:base-url}+ :config {:base-url "http://vllm.internal:8000/v1"}}]+ (llm.provider/connections))))+ (testing "and a key on its own does not: it authenticates nothing without a server to send it to"+ (mt/with-temp-env-var-value! [mb-llm-vllm-api-key "local-dev-key"]+ (is (= [] (llm.provider/connections)))))))+ (testing "a metabase/ reference pinned by the environment synthesizes the managed connection it names"+ (mt/with-temporary-setting-values [llm-providers []]+ (mt/with-temp-env-var-value! [mb-llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"]+ (is (=? [{:key "metabase" :type "metabase" :source :env}]+ (llm.provider/connections)))+ (is (=? {:connection-key "metabase" :type "anthropic" :ai-proxy? true}+ (llm.provider/resolve-model-ref "metabase/anthropic/claude-sonnet-4-6")))))))++(deftest connections-shadowed-by-the-environment-test(testing "the environment shadows a stored connection with the same key field by field, not wholesale"(mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic"{:api-key "sk-ant-db"@@ -296,18 +332,7 @@(mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic" {:api-key "sk-ant-db"})]](mt/with-temp-env-var-value! [mb-llm-anthropic-api-base-url "https://env.example.com"](is (= {:api-key "sk-ant-db" :base-url "https://env.example.com"}- (llm.provider/credentials "anthropic"))))))- (testing "a metabase/ reference pinned by the environment synthesizes the managed connection it names"- (mt/with-temporary-setting-values [llm-providers []]- (mt/with-temp-env-var-value! [mb-llm-metabot-provider "metabase/anthropic/claude-sonnet-4-6"]- (is (=? [{:key "metabase" :type "metabase" :source :env}]- (llm.provider/connections)))- (is (=? {:connection-key "metabase" :type "anthropic" :ai-proxy? true}- (llm.provider/resolve-model-ref "metabase/anthropic/claude-sonnet-4-6"))))))- (testing "the whole stored list is read-only when it comes from the environment"- (mt/with-temp-env-var-value! [mb-llm-providers "[{\"key\":\"anthropic\",\"type\":\"anthropic\",\"name\":\"Anthropic\",\"config\":{\"api-key\":\"sk-ant-env\"}}]"]- (is (= [["anthropic" :env]]- (map (juxt :key :source) (llm.provider/connections)))))))+ (llm.provider/credentials "anthropic")))))))(deftest stored-connections-keeps-a-connection-the-environment-shadows-test(testing (str "The stored list keeps the credentials the environment shadows, so writes rebuild from here and "@@ -433,7 +458,8 @@"so a type without a decided logo fails to compile. Nothing links the two, so adding a type here ""without updating them ships a provider that silently falls back to the generic icon. Update ""both, then this list.")- (is (= #{"anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "metabase"}+ (is (= #{"anthropic" "openai" "openrouter" "mistral" "zai" "moonshot" "google" "azure" "bedrock" "vllm"+ "metabase"}(into #{} (map :type) (llm.provider/provider-types))))))(deftest ^:parallel provider-types-test@@ -450,6 +476,7 @@(is (= #{:api-key} (llm.provider/secret-field-keys "anthropic")))(is (= #{:api-key} (llm.provider/secret-field-keys "azure")))(is (= #{:access-key-id :secret-access-key :session-token} (llm.provider/secret-field-keys "bedrock")))+ (is (= #{:api-key} (llm.provider/secret-field-keys "vllm")))(testing "google's service account key is a file field, but it is the whole credential"(is (= #{:service-account-key :oauth-access-token} (llm.provider/secret-field-keys "google"))))(is (= #{} (llm.provider/secret-field-keys "metabase"))))@@ -464,6 +491,9 @@;; azure's models are deployment names the admin chooses, so there is nothing to default to"azure" nil"bedrock" "anthropic.claude-opus-4-8"+ ;; nor is there for vLLM, which serves whatever the operator loaded: connecting adopts the model+ ;; its probe exercised+ "vllm" nil"metabase" "anthropic/claude-sonnet-4-6"}(into {} (map (juxt :type #(llm.provider/default-model (:type %)))) (llm.provider/provider-types))))(is (nil? (llm.provider/default-model "evilai"))))@@ -479,6 +509,8 @@"google" nil"azure" nil"bedrock" "anthropic.claude-haiku-4-5"+ ;; a vLLM server serves the one model the operator loaded, so there is no cheaper tier to fall back to+ "vllm" nil"metabase" nil}(into {} (map (juxt :type #(llm.provider/mini-model (:type %)))) (llm.provider/provider-types))))(is (nil? (llm.provider/mini-model "evilai"))))diff --git a/test/metabase/llm/test_util.clj b/test/metabase/llm/test_util.cljindex 09be007ee17..6f07e268447 100644--- a/test/metabase/llm/test_util.clj+++ b/test/metabase/llm/test_util.clj@@ -23,6 +23,7 @@"bedrock" {:access-key-id "AKIAIOSFODNN7EXAMPLE":secret-access-key "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY":region "us-east-1"}+ "vllm" {:base-url "http://vllm.internal:8000/v1"}"metabase" {}})(defn connectiondiff --git a/test/metabase/metabot/self/openai/chat_completions_test.clj b/test/metabase/metabot/self/openai/chat_completions_test.cljindex 0464f1df81e..ae5d6cc0721 100644--- a/test/metabase/metabot/self/openai/chat_completions_test.clj+++ b/test/metabase/metabot/self/openai/chat_completions_test.clj@@ -308,6 +308,19 @@#"Mistral returned an unexpected model list response"(chat-completions/models-catalog "Mistral" {:status 200 :body body})))))))+(deftest ^:parallel models-catalog-appends-a-caller-supplied-detail-test+ (testing "a supplied detail is appended to the message"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"vLLM returned an unexpected model list response\. Check http://vllm\.internal:8000/v1\."+ (chat-completions/models-catalog "vLLM" {:status 200 :body {:object "list"}}+ {:detail "Check http://vllm.internal:8000/v1."}))))+ (testing "and a caller that supplies none gets the bare message"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Mistral returned an unexpected model list response$"+ (chat-completions/models-catalog "Mistral" {:status 200 :body {:object "list"}} nil)))))+(deftest ^:parallel models-catalog-error-is-tagged-api-error-without-status-test(testing "the thrown error is tagged :api-error so rethrow-api-error! passes it through, and carries no status";; `metabase.metabot.api/provider-client-error?` renders any 4xx :api-error under the admin API-keydiff --git a/test/metabase/metabot/self/openrouter_test.clj b/test/metabase/metabot/self/openrouter_test.cljindex 81a1eac711a..c3414f9f935 100644--- a/test/metabase/metabot/self/openrouter_test.clj+++ b/test/metabase/metabot/self/openrouter_test.clj@@ -72,6 +72,44 @@:input [{:role :user :content "hi"}]})](is (= ["user"] (map :role (:messages body)))))))+;;; ──────────────────────────────────────────────────────────────────+;;; Temperature gating+;;; ──────────────────────────────────────────────────────────────────++(defn- request-body-temperature+ [model]+ (:temperature (openrouter/openrouter-request-body {:model model+ :input [{:role :user :content "hi"}]+ :temperature 0.3})))++(deftest ^:parallel request-body-drops-temperature-for-models-that-reject-it-test+ (testing "the GPT-5 and o-series families take no explicit temperature"+ (doseq [model ["openai/gpt-5.6-sol" "openai/gpt-5.5-pro" "openai/gpt-5.4" "openai/gpt-5.4-mini"+ "openai/o1" "openai/o3-mini"]]+ (testing model+ (is (nil? (request-body-temperature model))))))+ (testing "neither does current-generation Claude, whose OpenRouter ids use dots for the minor version"+ (doseq [model ["anthropic/claude-fable-5" "anthropic/claude-opus-5" "anthropic/claude-opus-4.8"+ "anthropic/claude-opus-4.7" "anthropic/claude-sonnet-5"]]+ (testing model+ (is (nil? (request-body-temperature model)))))))++(deftest ^:parallel request-body-keeps-temperature-for-models-that-accept-it-test+ (testing "every other whitelisted model still gets the profile's temperature"+ (doseq [model ["anthropic/claude-opus-4.6" "anthropic/claude-opus-4.5" "anthropic/claude-opus-4.1"+ "anthropic/claude-sonnet-4.6" "anthropic/claude-sonnet-4.5" "anthropic/claude-haiku-4.5"+ "deepseek/deepseek-v4-pro" "mistralai/mistral-medium-3-5" "z-ai/glm-5.2"]]+ (testing model+ (is (= 0.3 (request-body-temperature model)))))))++(deftest ^:parallel request-body-omits-temperature-when-none-is-supplied-test+ (testing "a request with no temperature is unchanged either way"+ (doseq [model ["openai/gpt-5.4" "anthropic/claude-haiku-4.5"]]+ (testing model+ (is (not (contains? (openrouter/openrouter-request-body {:model model+ :input [{:role :user :content "hi"}]})+ :temperature)))))))+;;; ──────────────────────────────────────────────────────────────────;;; openrouter-request-body tool_choice tests;;; ──────────────────────────────────────────────────────────────────diff --git a/test/metabase/metabot/self/vllm_test.clj b/test/metabase/metabot/self/vllm_test.cljnew file mode 100644index 00000000000..43b99d27f8b--- /dev/null+++ b/test/metabase/metabot/self/vllm_test.clj@@ -0,0 +1,1098 @@+(ns metabase.metabot.self.vllm-test+ (:require+ [clj-http.client :as http]+ [clojure.test :refer :all]+ [metabase.llm.settings :as llm.settings]+ [metabase.metabot.self :as self]+ [metabase.metabot.self.core :as self.core]+ [metabase.metabot.self.debug :as debug]+ [metabase.metabot.self.openai.chat-completions :as chat-completions]+ [metabase.metabot.self.vllm :as vllm]+ [metabase.metabot.test-util :as metabot.tu]+ [metabase.test :as mt]+ [metabase.util.json :as json])+ (:import+ (java.net ConnectException SocketTimeoutException)+ (java.util.concurrent CountDownLatch TimeUnit)))++(set! *warn-on-reflection* true)++(def ^:private base-url "http://vllm.internal:8000/v1")++(def ^:private credentials+ "What a resolved vLLM connection hands the adapter: adapters read credentials only, never settings."+ {:base-url base-url})++(def ^:private keyed-credentials+ "A connection to a server started with --api-key."+ (assoc credentials :api-key "local-dev-key"))++(def ^:private reasoning-credentials+ "A connection whose connect-time probe found the served model streaming its reasoning."+ (assoc credentials vllm/reasoning-config-key "true"))++;;; ──────────────────────────────────────────────────────────────────+;;; vllm-request-body+;;; ──────────────────────────────────────────────────────────────────++(deftest ^:parallel request-body-applies-default-max-tokens-test+ (testing "an explicit max_tokens is always sent — without one vLLM falls back to the whole remaining context window"+ (is (= (llm.settings/llm-max-tokens)+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]}))))))++(deftest ^:parallel request-body-caller-max-tokens-wins-test+ (testing "a caller-supplied max-tokens is not overridden by the default"+ (is (= 128+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :max-tokens 128}))))))++(deftest ^:parallel request-body-raises-max-tokens-for-a-forced-tool-call-test+ (testing "a forced tool call gets the token floor even when the caller asked for less"+ (testing "schema"+ (is (= 2048+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :schema {:type "object"}+ :max-tokens 128})))))+ (testing "tool_choice required"+ (is (= 2048+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :tools [(metabot.tu/get-time-tool)]+ :tool_choice "required"+ :max-tokens 128})))))))++(deftest ^:parallel request-body-floor-never-lowers-a-ceiling-test+ (testing "a caller-supplied ceiling above the floor is left alone"+ (is (= 8192+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :schema {:type "object"}+ :max-tokens 8192}))))))++(deftest ^:parallel request-body-floor-applies-only-to-forced-tool-calls-test+ (testing "an unforced request keeps the caller's ceiling — the floor exists for thinking that precedes a tool call"+ (is (= 128+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :tools [(metabot.tu/get-time-tool)]+ :tool_choice "auto"+ :max-tokens 128}))))))++(deftest ^:parallel request-body-raises-max-tokens-for-a-reasoning-model-test+ (testing "the agent path forces nothing and supplies no ceiling, so a reasoning model would otherwise get+ the shared 4096 default for thinking, answer, and tool call combined"+ (is (= 16384+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials reasoning-credentials}))))+ (testing "and a model the probe found does not reason keeps the default"+ (is (= (llm.settings/llm-max-tokens)+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials credentials})))))))++(deftest ^:parallel request-body-reasoning-floor-outranks-the-forced-tool-call-floor-test+ (testing "a reasoning model has to clear its thinking before the forced call, so the higher floor wins"+ (is (= 16384+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :schema {:type "object"}+ :max-tokens 128+ :credentials reasoning-credentials}))))))++(deftest ^:parallel request-body-reasoning-floor-never-lowers-a-ceiling-test+ (testing "a caller asking for more than the floor is left alone"+ (is (= 32000+ (:max_tokens (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :max-tokens 32000+ :credentials reasoning-credentials}))))))++(deftest ^:parallel request-body-no-default-model-test+ (testing "the model is never defaulted — a vLLM server's model name is whatever the operator loaded"+ (is (nil? (:model (vllm/vllm-request-body {:input [{:role :user :content "hi"}]}))))))++(deftest ^:parallel request-body-system-and-stream-test+ (testing "the system prompt leads the messages and the request streams with usage included"+ (let [body (vllm/vllm-request-body {:model "vllm-test"+ :system "You are a helpful assistant."+ :input [{:role :user :content "hi"}]})]+ (is (= {:role "system" :content "You are a helpful assistant."}+ (-> body :messages first)))+ (is (true? (:stream body)))+ (is (= {:include_usage true} (:stream_options body))))))++(deftest ^:parallel request-body-tools-test+ (testing "tools are sent in OpenAI function format with tool_choice auto"+ (is (=? {:tools [{:type "function"+ :function {:name "get-time"}}]+ :tool_choice "auto"}+ (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :tools [(metabot.tu/get-time-tool)]})))))++(deftest ^:parallel request-body-schema-forces-structured-output-test+ (testing "a schema forces a structured_output tool call"+ (is (=? {:tools [{:type "function"+ :function {:name "structured_output"+ :parameters {:type "object"}}}]+ :tool_choice "required"}+ (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :schema {:type "object"+ :properties {:title {:type "string"}}}})))))++(deftest ^:parallel request-body-temperature-test+ (testing "a caller's temperature passes through"+ (is (= 0.7+ (:temperature (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :temperature 0.7})))))+ (testing "temperature 0 is honoured rather than treated as absent"+ (is (= 0+ (:temperature (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :temperature 0}))))))++(deftest ^:parallel request-body-supplies-a-default-temperature-test+ (testing "a caller that supplies none gets the adapter default rather than vLLM's own 1.0"+ (is (= @#'vllm/default-temperature+ (:temperature (vllm/vllm-request-body {:model "vllm-test"+ :input [{:role :user :content "hi"}]}))))))++;;; ──────────────────────────────────────────────────────────────────+;;; Streaming chunk conversion+;;;+;;; Recorded verbatim from vLLM 0.26.0 serving Qwen3-14B with+;;; `--tool-call-parser hermes`.+;;; ──────────────────────────────────────────────────────────────────++(def ^:private recorded-tool-call-stream+ (let [msg-id "chatcmpl-ba409270f617984a"+ arg (fn [s] {:id msg-id :model "vllm-test"+ :choices [{:index 0 :delta {:content nil :tool_calls [{:index 0 :function {:arguments s}}]}+ :finish_reason nil}]})]+ (into [{:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:role "assistant" :content ""} :finish_reason nil}]}+ {:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:content "<think>"} :finish_reason nil}]}+ {:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:content "\n\n"} :finish_reason nil}]}+ {:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:content "</think>"} :finish_reason nil}]}+ {:id msg-id :model "vllm-test" :choices [{:index 0 :delta {:content "\n\n"} :finish_reason nil}]}+ {:id msg-id :model "vllm-test"+ :choices [{:index 0+ :delta {:content nil+ :tool_calls [{:id "chatcmpl-tool-acf1cddb0052df26"+ :type "function"+ :index 0+ :function {:name "record_table_name"}}]}+ :finish_reason nil}]}]+ cat+ [(map arg ["{\"" "table" "_name" "\":" " \"" "orders" "\"}"])+ [{:id msg-id :model "vllm-test" :choices [{:index 0 :delta {} :finish_reason "tool_calls"}]}+ {:id msg-id :model "vllm-test" :choices []+ :usage {:prompt_tokens 163 :total_tokens 189 :completion_tokens 26}}]])))++(deftest ^:parallel recorded-streamed-tool-call-conv-test+ (testing "a tool call whose id arrives only on the first delta accumulates into one tool-input"+ (let [parts (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)) recorded-tool-call-stream)]+ (is (=? {:type :tool-input+ :id "chatcmpl-tool-acf1cddb0052df26"+ :function "record_table_name"+ :arguments {:table_name "orders"}}+ (first (filter #(= :tool-input (:type %)) parts)))))))++(deftest ^:parallel recorded-usage-has-no-cache-details-test+ (testing "vLLM reports no prompt_tokens_details, so the cache columns come through as zero rather than throwing"+ (let [parts (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)) recorded-tool-call-stream)]+ (is (=? {:type :usage+ :model "vllm-test"+ :usage {:promptTokens 163+ :completionTokens 26+ :cacheCreationTokens 0+ :cacheReadTokens 0}}+ (first (filter #(= :usage (:type %)) parts)))))))++(deftest ^:parallel think-tags-reach-the-text-stream-test+ (testing "without --reasoning-parser a model's thinking arrives in delta.content and streams as assistant text"+ (let [text (->> (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)) recorded-tool-call-stream)+ (filter #(= :text (:type %)))+ (map :text)+ (apply str))]+ (is (re-find #"<think>" text)))))++;;; Recorded from vLLM 0.26.0 serving Qwen3-14B with `--reasoning-parser qwen3`. The shape that+;;; could not be guessed: the field is `reasoning`, not `reasoning_content` — 0.26 renamed it and+;;; documents the old spelling as deprecated. The opening chunk carries `content ""`, which must not+;;; open a text block ahead of the reasoning.+(def ^:private reasoning-stream+ [{:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"+ :choices [{:index 0 :delta {:role "assistant" :content ""} :finish_reason nil}]}+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"+ :choices [{:index 0 :delta {:reasoning "\nOkay"} :finish_reason nil}]}+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"+ :choices [{:index 0 :delta {:reasoning ", the user asks 2+2."} :finish_reason nil}]}+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"+ :choices [{:index 0 :delta {:content "Four"} :finish_reason nil}]}+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"+ :choices [{:index 0 :delta {:content "."} :finish_reason nil}]}+ {:id "chatcmpl-b3043ba7f7b178ce" :model "vllm-test"+ :choices [{:index 0 :delta {} :finish_reason "stop"}]}])++;;; The deprecated spelling, still emitted by older vLLM builds and other OpenAI-compatible servers.+(def ^:private deprecated-reasoning-stream+ (mapv (fn [chunk]+ (update-in chunk [:choices 0 :delta]+ (fn [delta]+ (if-let [r (:reasoning delta)]+ (-> delta (dissoc :reasoning) (assoc :reasoning_content r))+ delta))))+ reasoning-stream))++(deftest ^:parallel reasoning-becomes-reasoning-parts-test+ (testing "with --reasoning-parser, thinking streams as reasoning and the answer still streams as text"+ (doseq [[spelling stream] {"reasoning" reasoning-stream+ "reasoning_content" deprecated-reasoning-stream}]+ (testing spelling+ (let [parts (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)) stream)]+ (is (= ["\nOkay, the user asks 2+2."] (map :text (filter #(= :reasoning (:type %)) parts))))+ (is (= ["Four."] (map :text (filter #(= :text (:type %)) parts)))))))))++(deftest ^:parallel reasoning-chunks-are-bracketed-test+ (testing "the raw chunk stream brackets reasoning the way self.core expects to group it"+ (let [chunks (into [] (vllm/vllm->aisdk-chunks-xf) reasoning-stream)+ by-type (group-by :type chunks)]+ (testing "the opening `content \"\"` chunk does not open a text block ahead of the reasoning"+ (is (= [:start :reasoning-start :reasoning-delta :reasoning-delta :reasoning-end+ :text-start :text-delta :text-delta :text-end]+ (mapv :type chunks))))+ (testing "every reasoning chunk shares one id, so aisdk-xf joins them into a single part"+ (is (= 1 (count (into #{} (map :id) (mapcat by-type [:reasoning-start :reasoning-delta :reasoning-end])))))))))++(deftest ^:parallel reasoning-forwarding-is-opt-in-test+ (testing "the shared xf still drops reasoning for the adapters that did not opt in"+ (doseq [[spelling stream] {"reasoning" reasoning-stream+ "reasoning_content" deprecated-reasoning-stream}]+ (testing spelling+ (let [parts (into [] (comp (chat-completions/chat-completions->aisdk-chunks-xf) (self.core/aisdk-xf))+ stream)]+ (is (empty? (filter #(= :reasoning (:type %)) parts)))+ (is (= ["Four."] (map :text (filter #(= :text (:type %)) parts)))))))))++;;; Recorded verbatim from vLLM 0.26.0 serving vllm-test with+;;; `--served-model-name vllm-test --max-model-len 32768 --enable-auto-tool-choice --tool-call-parser hermes --reasoning-parser qwen3`.+;;; Inert envelope fields (created, logprobs, object, prompt_text, prompt_token_ids, service_tier, system_fingerprint, token_ids) dropped for size;+;;; the full capture is under `logs/ai/requests/`.+;;; 155 chunks captured, 22 kept: each run of reasoning deltas is truncated+;;; to its first 2 (133 reasoning dropped). Nothing structural is removed.+(def ^:private recorded-parallel-tool-call-stream+ [{:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices [{:index 0, :delta {:role "assistant", :content ""}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices [{:index 0, :delta {:reasoning "\n"}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices [{:index 0, :delta {:reasoning "Okay"}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices [{:index 0, :delta {:content "\n\n"}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0,+ :delta+ {:content nil,+ :tool_calls+ [{:id "chatcmpl-tool-96102e0493e264b7", :type "function", :index 0, :function {:name "record_table_name"}}]},+ :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "{\""}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "table"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "_name"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "\":"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments " \""}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "orders"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 0, :function {:arguments "\"}"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0,+ :delta+ {:content nil,+ :tool_calls+ [{:id "chatcmpl-tool-b34f3b547d2105f2", :type "function", :index 1, :function {:name "record_column_name"}}]},+ :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "{\""}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "column"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "_name"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "\":"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments " \""}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "total"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices+ [{:index 0, :delta {:content nil, :tool_calls [{:index 1, :function {:arguments "\"}"}}]}, :finish_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices [{:index 0, :delta {}, :finish_reason "tool_calls", :stop_reason nil}]}+ {:id "chatcmpl-8ad4c404bbb917ae",+ :model "vllm-test",+ :choices [],+ :usage {:prompt_tokens 244, :total_tokens 426, :completion_tokens 182}}])++(deftest ^:parallel recorded-parallel-tool-calls-become-separate-parts-test+ (testing "two tool calls in one assistant message accumulate into two parts, each with its own arguments"+ (let [calls (->> recorded-parallel-tool-call-stream+ (into [] (comp (vllm/vllm->aisdk-chunks-xf) (self.core/aisdk-xf)))+ (filter #(= :tool-input (:type %))))]+ (is (= [{:id "chatcmpl-tool-96102e0493e264b7" :function "record_table_name"+ :arguments {:table_name "orders"}}+ {:id "chatcmpl-tool-b34f3b547d2105f2" :function "record_column_name"+ :arguments {:column_name "total"}}]+ (mapv #(select-keys % [:id :function :arguments]) calls))))))++(deftest ^:parallel recorded-parallel-tool-calls-do-not-interleave-test+ (testing "the xf keys transitions on the tool-call id and never reads index, which is safe only because+ hermes emits each call's deltas as one contiguous run"+ (let [deltas (for [chunk recorded-parallel-tool-call-stream+ choice (:choices chunk)+ tc (get-in choice [:delta :tool_calls])]+ tc)]+ (testing "index arrives in contiguous runs, not interleaved"+ (is (= [0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1] (mapv :index deltas))))+ (testing "id arrives on each call's opening delta only — a provider repeating it would lose the+ arguments, since neither the start branch nor the argument-delta branch would fire"+ (is (= [0 1] (keep #(when (:id %) (:index %)) deltas)))))))++;;; ──────────────────────────────────────────────────────────────────+;;; Auth+;;; ──────────────────────────────────────────────────────────────────++(defn- captured-chat-request!+ "Run `vllm-raw` against a stubbed transport and return the request it issued."+ [opts]+ (let [captured (atom nil)]+ (with-redefs [self.core/sse-reducible (fn [_] (reify clojure.lang.IReduceInit+ (reduce [_ _rf init] init)))+ debug/capture-stream (fn [r _] r)+ http/request (fn [req] (reset! captured req) {:body nil})]+ (vllm/vllm-raw opts))+ @captured))++(deftest vllm-auth-sends-bearer-token-when-configured-test+ (testing "the connection's API key becomes an Authorization header against its base URL"+ (is (=? {:method :post+ :url "http://vllm.internal:8000/v1/chat/completions"+ :headers {"Authorization" "Bearer local-dev-key"}+ :body string?}+ (captured-chat-request! {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials keyed-credentials})))))++(deftest vllm-auth-omits-authorization-without-a-key-test+ (testing "a keyless server is a complete configuration — no Authorization header, and no missing-key error"+ (let [req (captured-chat-request! {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials credentials})]+ (is (= "http://vllm.internal:8000/v1/chat/completions" (:url req)))+ (is (not (contains? (:headers req) "Authorization"))))))++(deftest vllm-sets-a-socket-timeout-test+ (testing "generation requests carry the vLLM socket timeout and the shared connection timeout"+ (mt/with-temporary-setting-values [llm.settings/llm-vllm-request-timeout-ms 300000]+ (is (=? {:socket-timeout 300000+ :connection-timeout (llm.settings/llm-connection-timeout-ms)}+ (captured-chat-request! {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials credentials}))))))++(deftest vllm-auth-missing-base-url-test+ (testing "a connection carrying no base URL fails with a message naming vLLM, not a clj-http error about a+ relative URL"+ (with-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No vLLM base URL is set"+ (vllm/vllm-raw {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials {:api-key "local-dev-key"}})))+ (is (= :base-url-missing+ (:error-code (try (vllm/list-models {:credentials {:api-key "local-dev-key"}})+ (catch clojure.lang.ExceptionInfo e (ex-data e)))))))))++(deftest vllm-missing-model-test+ (testing "a blank model fails with a vLLM-specific message before any request is made"+ (with-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No vLLM model is set"+ (vllm/vllm-raw {:input [{:role :user :content "hi"}] :credentials credentials}))))))++(deftest vllm-ai-proxy-unsupported-test+ (testing "ai-proxy? throws before credentials are even consulted"+ (with-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"AI proxy is not supported for vLLM"+ (vllm/vllm-raw {:model "vllm-test" :input [{:role :user :content "hi"}] :ai-proxy? true})))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"AI proxy is not supported for vLLM"+ (vllm/list-models {:ai-proxy? true}))))))++(deftest vllm-stream-socket-timeout-is-not-retryable-test+ (testing "a socket timeout while consuming the stream surfaces as a tagged vLLM error, not a raw SocketTimeoutException"+ (with-redefs [self.core/sse-reducible (fn [_]+ (reify clojure.lang.IReduceInit+ (reduce [_ _rf _init]+ (throw (SocketTimeoutException. "Read timed out")))))+ debug/capture-stream (fn [r _] r)+ http/request (fn [_] {:body nil})]+ (let [e (try (into [] (vllm/vllm-raw {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials credentials}))+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= :vllm-timeout (:error-code (ex-data e))))+ (is (re-find #"stopped responding" (ex-message e)))+ (testing "and is not retryable"+ (is (false? (#'self/retryable-error? e))))))))++(deftest vllm-stream-connection-failure-is-not-retryable-test+ (testing "a stream severed mid-body surfaces as a tagged vLLM error, not a raw IOException"+ (with-redefs [self.core/sse-reducible (fn [_]+ (reify clojure.lang.IReduceInit+ (reduce [_ _rf _init]+ (throw (java.io.IOException. "Connection reset")))))+ debug/capture-stream (fn [r _] r)+ http/request (fn [_] {:body nil})]+ (let [e (try (into [] (vllm/vllm-raw {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials credentials}))+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= :vllm-stream-interrupted (:error-code (ex-data e))))+ (is (re-find #"interrupted before the response finished" (ex-message e)))+ (testing "and is not retryable"+ (is (false? (#'self/retryable-error? e))))))))++(deftest vllm-mid-stream-non-io-error-is-translated-test+ (testing "a mid-stream failure that is not IO gets the shared provider translation, the same as every+ other adapter — `io-guarded` composes with `reducible-with-api-errors` rather than replacing it"+ (with-redefs [self.core/sse-reducible (fn [_]+ (reify clojure.lang.IReduceInit+ (reduce [_ _rf _init]+ (throw (ex-info "clj-http: status 500"+ {:status 500 :body "{\"message\":\"boom\"}"})))))+ debug/capture-stream (fn [r _] r)+ http/request (fn [_] {:body nil})]+ (let [e (try (into [] (vllm/vllm-raw {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials credentials}))+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= :provider-api-error (:error-code (ex-data e))))+ (is (re-find #"vLLM returned an internal server error" (ex-message e)))))))++(deftest vllm-request-timeout-names-vllm-and-its-setting-test+ (testing "a timeout while establishing the request gets the same treatment the preflight already gives it,+ rather than `rethrow-api-error!`'s \"vllm API request failed: Read timed out\""+ (mt/with-temporary-setting-values [llm.settings/llm-vllm-request-timeout-ms 300000]+ (with-redefs [http/request (fn [_] (throw (SocketTimeoutException. "Read timed out")))]+ (let [e (try (vllm/vllm-raw {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials credentials})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= :vllm-timeout (:error-code (ex-data e))))+ (is (re-find #"did not respond within 300000ms" (ex-message e)))+ (is (re-find #"raise the vLLM request timeout" (ex-message e)))+ (testing "and is not retryable"+ (is (false? (#'self/retryable-error? e)))))))))++(deftest vllm-unreachable-server-names-the-base-url-test+ (testing "a refused connection is the base URL being wrong or the server being down — say which URL failed"+ (with-redefs [http/request (fn [_] (throw (ConnectException. "Connection refused")))]+ (let [e (try (vllm/vllm-raw {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials credentials})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= :vllm-unreachable (:error-code (ex-data e))))+ (is (re-find #"Could not reach the vLLM server at http://vllm\.internal:8000/v1" (ex-message e)))+ (testing "and is not retryable"+ (is (false? (#'self/retryable-error? e))))))))++(deftest vllm-http-errors-still-reach-the-status-specific-message-test+ (testing "the IOException catch runs first, so a non-2xx must still be translated by `vllm-error-msg`"+ (with-redefs [http/request (fn [_] (throw (ex-info "clj-http: status 401"+ {:status 401 :body "{\"message\":\"Unauthorized\"}"})))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"vLLM API key expired or invalid"+ (vllm/vllm-raw {:model "vllm-test"+ :input [{:role :user :content "hi"}]+ :credentials credentials}))))))++;;; ──────────────────────────────────────────────────────────────────+;;; list-models+;;; ──────────────────────────────────────────────────────────────────++;;; Recorded from vLLM 0.26.0 serving mlx-community/Qwen3-14B-4bit with+;;; `--served-model-name vllm-test vllm-test-alias qwen3-14b` (BOT-1930 L-10). Aliases are the cheapest+;;; multi-entry catalog: several ids over one loaded model. Note the key set — there is no `name`, so+;;; `display_name` always falls back to the served id on vLLM, and `parent` is present-and-nil rather+;;; than absent. `:permission` is dropped here; nothing reads it.+(def ^:private recorded-multi-model-catalog+ [{:id "vllm-test" :object "model" :created 1786676106 :owned_by "vllm"+ :root "mlx-community/Qwen3-14B-4bit" :parent nil :max_model_len 32768}+ {:id "vllm-test-alias" :object "model" :created 1786676106 :owned_by "vllm"+ :root "mlx-community/Qwen3-14B-4bit" :parent nil :max_model_len 32768}+ {:id "qwen3-14b" :object "model" :created 1786676106 :owned_by "vllm"+ :root "mlx-community/Qwen3-14B-4bit" :parent nil :max_model_len 32768}])++(deftest list-models-passes-the-catalog-through-test+ (testing "every served model is offered — there is no whitelist — in the order the server lists them"+ (mt/with-dynamic-fn-redefs [http/request (fn [req]+ (is (=? {:method :get+ :url "http://vllm.internal:8000/v1/models"+ :headers {"Authorization" "Bearer local-dev-key"}}+ req))+ {:status 200 :body {:data recorded-multi-model-catalog}})]+ (is (= {:models [{:id "vllm-test" :display_name "vllm-test"}+ {:id "vllm-test-alias" :display_name "vllm-test-alias"}+ {:id "qwen3-14b" :display_name "qwen3-14b"}]}+ (vllm/list-models {:credentials keyed-credentials}))))))++;;; Recorded from the same server started **without** `--served-model-name` (BOT-1930 L-10), which is+;;; how an operator who does not rename their deployment sees it: the catalog id is the Hugging Face+;;; repo id, slashes and all, and `root` equals `id` rather than sitting behind an alias.+(def ^:private recorded-no-alias-catalog+ [{:id "mlx-community/Qwen3-14B-4bit" :object "model" :created 1786676633 :owned_by "vllm"+ :root "mlx-community/Qwen3-14B-4bit" :parent nil :max_model_len 32768}])++(deftest list-models-keeps-slashes-in-a-served-model-id-test+ (testing "a Hugging Face repo id survives the listing intact — `llm-metabot-provider` stores it as+ `vllm/{id}`, so the model segment is the only one allowed to keep its slashes"+ (mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body {:data recorded-no-alias-catalog}})]+ (is (= {:models [{:id "mlx-community/Qwen3-14B-4bit" :display_name "mlx-community/Qwen3-14B-4bit"}]}+ (vllm/list-models {:credentials credentials}))))))++(deftest list-models-display-name-falls-back-to-the-served-id-test+ (testing "a `name` is honoured when present, though no vLLM build emits one — the tolerance is for the+ other OpenAI-compatible servers the same UI copy invites"+ (mt/with-dynamic-fn-redefs [http/request (fn [_]+ {:status 200+ :body {:data [{:id "some-finetune" :name "Some Finetune"}]}})]+ (is (= {:models [{:id "some-finetune" :display_name "Some Finetune"}]}+ (vllm/list-models {:credentials credentials}))))))++(deftest list-models-does-not-fall-back-to-the-single-provider-settings-test+ (testing "the connection's credentials are the only source — a setting configuring another connection+ is not a fallback"+ (mt/with-temporary-setting-values [llm.settings/llm-vllm-api-base-url "http://saved:8000/v1"+ llm.settings/llm-vllm-api-key "saved-key"]+ (mt/with-dynamic-fn-redefs [http/request (fn [req]+ (is (=? {:url "http://explicit:8000/v1/models"+ :headers {"Authorization" "Bearer explicit-key"}}+ req))+ {:status 200 :body {:data []}})]+ (is (= {:models []}+ (vllm/list-models {:credentials {:base-url "http://explicit:8000/v1"+ :api-key "explicit-key"}}))))+ (testing "and a connection without a base URL fails rather than borrowing the setting's"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"No vLLM base URL is set"+ (vllm/list-models {:credentials {:api-key "explicit-key"}})))))))++(deftest list-models-fails-closed-on-a-body-that-is-not-a-catalog-test+ (testing "a 2xx whose body carries no model list throws, naming the base URL"+ (doseq [body [{:status "ok" :service "some-other-thing"} {:object "list"} "<html>404</html>"]]+ (testing (str "body " (pr-str body))+ (mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body body})]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"vLLM returned an unexpected model list response.*http://vllm\.internal:8000/v1"+ (vllm/list-models {:credentials credentials})))))))+ (testing "a well-formed but empty catalog lists nothing and gets its own distinct message"+ (mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body {:object "list" :data []}})]+ (is (= {:models []} (vllm/list-models {:credentials credentials})))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"reachable but is not serving any models"+ (vllm/list-models {:credentials credentials :probe? true}))))))++(deftest list-models-fails-closed-before-probing-test+ (testing "a malformed catalog throws without issuing a probe request"+ (let [chat-requests (atom 0)]+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]+ (when-not (re-find #"/models$" (str url))+ (swap! chat-requests inc))+ {:status 200 :body {:status "ok"}})]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"vLLM returned an unexpected model list response"+ (vllm/list-models {:credentials credentials :probe? true})))+ (is (zero? @chat-requests))))))++(deftest list-models-401-maps-to-invalid-key-message-test+ (testing "a 401 surfaces as the canonical message naming --api-key"+ (mt/with-dynamic-fn-redefs [http/request (fn [_]+ (throw (ex-info "clj-http: status 401"+ {:status 401+ :body "{\"message\":\"Unauthorized\"}"})))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"vLLM API key expired or invalid"+ (vllm/list-models {:credentials (assoc credentials :api-key "wrong-key")}))))))++(deftest list-models-404-points-at-the-base-url-test+ (testing "a 404 tells the admin the base URL should end in /v1"+ (mt/with-dynamic-fn-redefs [http/request (fn [_]+ (throw (ex-info "clj-http: status 404" {:status 404 :body "not found"})))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"base URL should end in /v1"+ (vllm/list-models {:credentials {:base-url "http://vllm.internal:8000"}}))))))++(deftest list-models-unreachable-server-is-a-client-error-test+ (testing "a refused connection while fetching the catalog names the base URL and is tagged 400, so a+ mistyped base URL reaches the admin as the message instead of a 500"+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (ConnectException. "Connection refused")))]+ (let [e (try (vllm/list-models {:credentials credentials})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= {:api-error true :status-code 400 :error-code :vllm-unreachable}+ (select-keys (ex-data e) [:api-error :status-code :error-code])))+ (is (re-find #"Could not reach the vLLM server at http://vllm\.internal:8000/v1" (ex-message e)))))))++(deftest list-models-timeout-names-the-server-test+ (testing "the catalog fetch runs on the shared request timeout, so a stall there reports that budget+ rather than the vLLM generation one"+ (mt/with-temporary-setting-values [llm.settings/llm-request-timeout-ms 120000+ llm.settings/llm-vllm-request-timeout-ms 300000]+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (SocketTimeoutException. "Read timed out")))]+ (let [e (try (vllm/list-models {:credentials credentials})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= {:api-error true :status-code 400 :error-code :vllm-timeout}+ (select-keys (ex-data e) [:api-error :status-code :error-code])))+ (is (re-find #"http://vllm\.internal:8000/v1 did not respond within 120000ms" (ex-message e))))))))++;;; ──────────────────────────────────────────────────────────────────+;;; Preflight+;;; ──────────────────────────────────────────────────────────────────++(defn- probing-server+ "Stub `http/request` for the preflight path: `GET /models` returns `models`, and each+ `POST /chat/completions` returns whatever `choice-by-tool-choice` holds for the `tool_choice` it+ was sent.++ Dispatching on `tool_choice` is what lets the two probes disagree. They exercise different server+ capabilities — the tool-call parser under `auto`, guided decoding under `required` — so a stub that+ answers both identically can only ever reach `check-structured-output!` on the happy path."+ [models choice-by-tool-choice]+ (fn [{:keys [url body]}]+ (if (re-find #"/models$" (str url))+ {:status 200 :body {:data models}}+ (let [tool-choice (:tool_choice (json/decode+kw (str body)))]+ {:status 200 :body {:choices [(get choice-by-tool-choice tool-choice)]}}))))++(def ^:private tool-calling-message+ {:content ""+ :tool_calls [{:id "chatcmpl-tool-1"+ :type "function"+ :function {:name "record_table_name" :arguments "{\"table_name\": \"orders\"}"}}]})++(defn- probe-choice!+ "Run a preflight against a stub. The 2-arity answers both probes alike; the 3-arity lets them differ."+ ([models chat-choice]+ (probe-choice! models chat-choice chat-choice))+ ([models auto-choice required-choice]+ (mt/with-dynamic-fn-redefs [http/request (probing-server models {"auto" auto-choice+ "required" required-choice})]+ (vllm/list-models {:credentials credentials :probe? true}))))++(defn- probe!+ "[[probe-choice!]] for a server that runs to a natural stop, where only the message shape matters."+ [models chat-message]+ (probe-choice! models {:message chat-message :finish_reason "tool_calls"}))++(deftest preflight-passes-on-a-correctly-configured-server-test+ (testing "a server that returns a well-formed tool call passes and still lists its models"+ (is (= {:models [{:id "vllm-test" :display_name "vllm-test"}]+ :probed-model "vllm-test"+ :learned-config {vllm/reasoning-config-key "false"}}+ (probe! [{:id "vllm-test" :max_model_len 32768}] tool-calling-message)))))++(deftest preflight-reports-the-model-it-probed-test+ (testing "the probed model is named in the result rather than left to be re-derived from the listing —+ the connect path must adopt the model the contract checks actually ran against"+ (testing "the first catalog entry, in the order the server lists it"+ (is (= "vllm-test"+ (:probed-model (probe! recorded-multi-model-catalog tool-calling-message)))))+ (testing "including a slash-bearing repo id, which the connect path stores as `vllm/{id}`"+ (is (= "mlx-community/Qwen3-14B-4bit"+ (:probed-model (probe! recorded-no-alias-catalog tool-calling-message))))))+ (testing "and a listing that did not probe reports nothing"+ (mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body {:data [{:id "vllm-test"}]}})]+ (is (= [:models] (keys (vllm/list-models {:credentials credentials})))))))++(deftest preflight-skips-lora-adapters-when-picking-a-default-test+ (testing "an adapter registered with --lora-modules carries `parent`; adopting one would point Metabot+ at an adapter the admin never chose"+ (let [probed (atom nil)]+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body]}]+ (if (re-find #"/models$" (str url))+ {:status 200+ :body {:data [{:id "sql-lora"+ :parent "vllm-test"+ :max_model_len 32768}+ {:id "vllm-test"+ :parent nil+ :max_model_len 32768}]}}+ (do (reset! probed (:model (json/decode+kw (str body))))+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]+ (is (= "vllm-test" (:probed-model (vllm/list-models {:credentials credentials :probe? true}))))+ (is (= "vllm-test" @probed)))))+ (testing "an explicitly requested adapter is still honoured — the filter only picks the default"+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]+ (if (re-find #"/models$" (str url))+ {:status 200+ :body {:data [{:id "sql-lora" :parent "vllm-test" :max_model_len 32768}+ {:id "vllm-test" :max_model_len 32768}]}}+ {:status 200 :body {:choices [{:message tool-calling-message}]}}))]+ (is (= "sql-lora" (:probed-model (vllm/list-models {:credentials credentials+ :probe? true+ :model "sql-lora"})))))))++(deftest preflight-skipped-without-probe-flag-test+ (testing "without :probe? no generation request is made at all — listing models must stay cheap"+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]+ (when-not (re-find #"/models$" (str url))+ (throw (ex-info "preflight must not run on a plain listing" {})))+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}})]+ (is (= {:models [{:id "vllm-test" :display_name "vllm-test"}]}+ (vllm/list-models {:credentials credentials}))))))++(deftest preflight-rejects-a-prose-answer-test+ (testing "a server that answers with text instead of a tool call names the two flags that cause it"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"--enable-auto-tool-choice"+ (probe! [{:id "vllm-test" :max_model_len 32768}]+ {:content "I'll record the table name orders for you." :tool_calls []})))))++(deftest preflight-rejects-unparseable-tool-arguments-test+ (testing "a tool call whose arguments are not JSON points at a mismatched --tool-call-parser"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"--tool-call-parser"+ (probe! [{:id "vllm-test" :max_model_len 32768}]+ {:content ""+ :tool_calls [{:id "chatcmpl-tool-1"+ :type "function"+ :function {:name "record_table_name" :arguments "table_name=orders"}}]})))))++(deftest preflight-rejects-leaked-reasoning-test+ (testing "reasoning arriving as chat text points at --reasoning-parser, even when the tool call itself is fine"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"--reasoning-parser"+ (probe! [{:id "vllm-test" :max_model_len 32768}]+ (assoc tool-calling-message :content "<think>\nThe user wants orders.\n</think>\n\n"))))))++(deftest preflight-rejects-a-context-window-that-is-too-small-test+ (testing "a model served under the context floor is rejected before any generation request"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"too small for Metabot"+ (probe! [{:id "vllm-test" :max_model_len 4096}] tool-calling-message)))))++(deftest preflight-rejects-a-server-with-no-models-test+ (testing "a reachable server serving nothing is a configuration error, not an empty dropdown"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"not serving any models"+ (probe! [] tool-calling-message)))))++(deftest preflight-probes-the-requested-model-test+ (testing "the probe targets the requested model, not merely the first one served"+ (let [probed (atom nil)]+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body]}]+ (if (re-find #"/models$" (str url))+ {:status 200 :body {:data [{:id "first" :max_model_len 32768}+ {:id "second" :max_model_len 32768}]}}+ (do (reset! probed (re-find #"second" (str body)))+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]+ (vllm/list-models {:credentials credentials :probe? true :model "second"})+ (is (= "second" @probed))))))++(deftest preflight-rejects-a-model-the-server-does-not-serve-test+ (testing "a requested model absent from the catalog fails and names what is served, rather than probing something else"+ (let [generated? (atom false)]+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]+ (if (re-find #"/models$" (str url))+ {:status 200 :body {:data [{:id "served-a" :max_model_len 32768}+ {:id "served-b" :max_model_len 32768}]}}+ (do (reset! generated? true)+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"not serving missing-model. It is serving: served-a, served-b"+ (vllm/list-models {:credentials credentials :probe? true :model "missing-model"})))+ (is (false? @generated?)+ "no generation request is issued for a model the server cannot run")))))++(deftest preflight-reports-a-thinking-budget-overrun-test+ (testing "a reasoning model that never stops thinking is named as such, not as a server missing tool-calling flags"+ (let [e (try (probe-choice! [{:id "vllm-test" :max_model_len 32768}]+ {:message {:content nil+ :reasoning "Okay, the user wants me to record a table name, so"+ :tool_calls []}+ :finish_reason "length"})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (re-find #"reasoning without calling a tool" (ex-message e)))+ (is (not (re-find #"--enable-auto-tool-choice" (ex-message e)))+ "the flags this server needs are already set, so naming them would send the admin the wrong way"))))++;;; The `content` below is the shape a live vLLM 0.26.0 returns for a tool call cut off at+;;; `max_tokens`, serving mlx-community/Qwen3-14B-4bit with `--enable-auto-tool-choice+;;; --tool-call-parser hermes` and `chat_template_kwargs {enable_thinking false}` (BOT-1930 L-7).+;;; The parser extracts from complete output, so without its closing `</tool_call>` the call yields no+;;; `tool_calls` entry at all and the raw text stays in `content`.+(deftest preflight-reports-a-truncated-tool-call-as-a-ceiling-overrun-test+ (testing "a generation cut off at the ceiling names the ceiling rather than the tool-calling flags,+ which the unterminated sentinel in the content shows are working"+ (let [e (try (probe-choice! [{:id "vllm-test" :max_model_len 32768}]+ {:message {:content "<tool_call>\n{\"name\": \"record_table_name\", \"arguments\": {\""+ :tool_calls []}+ :finish_reason "length"})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (re-find #"before completing a tool call" (ex-message e)))+ (is (re-find #"2048 token" (ex-message e)))+ (is (not (re-find #"--enable-auto-tool-choice" (ex-message e)))))))++(deftest preflight-reports-truncated-prose-as-a-ceiling-overrun-test+ (testing "prose truncated at the ceiling gets the same verdict — finish_reason is the only signal+ separating it from a truncated tool call"+ (let [e (try (probe-choice! [{:id "vllm-test" :max_model_len 32768}]+ {:message {:content "Sure! To record the table name I would first need to"+ :tool_calls []}+ :finish_reason "length"})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (re-find #"before completing a tool call" (ex-message e))))))++(deftest preflight-probe-timeout-is-capped-test+ (testing "a probe runs on its own budget, capped below the inference timeout"+ (mt/with-temporary-setting-values [llm.settings/llm-vllm-request-timeout-ms 600000]+ (let [socket-timeouts (atom [])]+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url socket-timeout]}]+ (if (re-find #"/models$" (str url))+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}+ (do (swap! socket-timeouts conj socket-timeout)+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]+ (vllm/list-models {:credentials credentials :probe? true})+ (is (= #{120000} (set @socket-timeouts))))))))++(deftest preflight-surfaces-a-probe-timeout-test+ (testing "a probe that times out says the server is too slow, not that the request failed"+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]+ (if (re-find #"/models$" (str url))+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}+ (throw (SocketTimeoutException. "Read timed out"))))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"did not answer the connection test"+ (vllm/list-models {:credentials credentials :probe? true}))))))++(deftest preflight-probes-run-concurrently-test+ (testing "both probes are in flight at once, so a slow server costs one probe budget rather than two"+ (let [arrived (CountDownLatch. 2)]+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]+ (if (re-find #"/models$" (str url))+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}+ (do (.countDown arrived)+ ;; Only satisfiable if the other probe is+ ;; already in flight.+ (is (.await arrived 10 TimeUnit/SECONDS))+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]+ (vllm/list-models {:credentials credentials :probe? true})))))++(deftest preflight-reports-whether-the-probed-model-reasons-test+ (testing "the probe is the only place this is knowable — /v1/models carries no reasoning field, so what it+ saw is reported back for the connection to record"+ ;; Recorded shape: vLLM 0.26 puts it on the non-streaming message under `reasoning`.+ (is (= {vllm/reasoning-config-key "true"}+ (:learned-config+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]+ {:message (assoc tool-calling-message+ :reasoning "\nOkay, the user wants me to record \"orders\".")+ :finish_reason "tool_calls"})))))+ (testing "and under the deprecated spelling older servers still use"+ (is (= {vllm/reasoning-config-key "true"}+ (:learned-config+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]+ {:message (assoc tool-calling-message+ :reasoning_content "Older vLLM builds spell it this way.")+ :finish_reason "tool_calls"})))))+ (testing "a model that answers without reasoning reports false"+ (is (= {vllm/reasoning-config-key "false"}+ (:learned-config (probe! [{:id "vllm-test" :max_model_len 32768}] tool-calling-message)))))+ (testing "and a connection carrying the flag is what the request body reads it from"+ (is (true? (vllm/reasoning-connection? {vllm/reasoning-config-key "true"})))+ (is (false? (vllm/reasoning-connection? {vllm/reasoning-config-key "false"})))+ (is (false? (vllm/reasoning-connection? {})))+ (testing "including the JSON boolean a hand-written llm-providers can hold"+ (is (true? (vllm/reasoning-connection? {vllm/reasoning-config-key true}))))))++(deftest preflight-reports-nothing-when-it-fails-test+ (testing "a failed probe throws rather than reporting a guess, so the connection records nothing"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"--enable-auto-tool-choice"+ (probe! [{:id "vllm-test" :max_model_len 32768}]+ {:content "I'll record the table name orders for you." :tool_calls []})))))++(deftest preflight-failures-are-surfaced-as-client-errors-test+ (testing "a preflight failure is tagged 400 so the admin sees the message instead of a 500"+ (let [data (try (probe! [{:id "vllm-test" :max_model_len 4096}] tool-calling-message)+ (catch clojure.lang.ExceptionInfo e (ex-data e)))]+ (is (= {:api-error true :status-code 400 :error-code :vllm-preflight-failed}+ (select-keys data [:api-error :status-code :error-code]))))))++(deftest preflight-rejects-a-server-that-cannot-force-a-tool-call-test+ (testing "a server whose --tool-call-parser is right but whose model cannot compile a grammar answers+ `auto` correctly and `required` with prose — ordinary chat looks fine while conversation+ titling and the whole sql profile break"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"did not honor a forced tool call"+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]+ {:message tool-calling-message :finish_reason "tool_calls"}+ {:message {:content "I can record the table name for you." :tool_calls []}+ :finish_reason "stop"})))))++(deftest preflight-reports-a-forced-tool-call-that-ran-out-of-budget-test+ (testing "a forced call truncated at the ceiling is named as such, not as a server refusing to honor it"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"reached the 2048 token connection-test ceiling"+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]+ {:message tool-calling-message :finish_reason "tool_calls"}+ {:message {:content "" :tool_calls []}+ :finish_reason "length"})))))++(deftest preflight-cancels-the-sibling-probe-on-failure-test+ (testing "the first verdict returns immediately and its sibling is cancelled — an abandoned future would+ keep generating against the operator's server after the admin already has a 400"+ (let [interrupted (promise)+ never (CountDownLatch. 1)]+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body]}]+ (if (re-find #"/models$" (str url))+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}+ (case (:tool_choice (json/decode+kw (str body)))+ "auto" {:status 200+ :body {:choices [{:message {:content "I'll record orders."+ :tool_calls []}}]}}+ "required" (try+ (.await never 10 TimeUnit/SECONDS)+ (deliver interrupted false)+ {:status 200 :body {:choices [{:message tool-calling-message}]}}+ (catch InterruptedException _+ (deliver interrupted true)+ (throw (SocketTimeoutException. "interrupted")))))))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"--enable-auto-tool-choice"+ (vllm/list-models {:credentials credentials :probe? true})))+ (is (true? (deref interrupted 10000 :never-cancelled)))))))++(deftest preflight-probe-request-shape-test+ (testing "both probes ask for the same trivial completion, non-streaming and deterministic — the ceiling+ is the one a forced tool call is guaranteed to be given at request time"+ (let [requests (atom [])]+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body] :as req}]+ (if (re-find #"/models$" (str url))+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}+ (do (swap! requests conj (assoc req :decoded (json/decode+kw (str body))))+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]+ (vllm/list-models {:credentials credentials :probe? true})+ (is (= 2 (count @requests)))+ (is (= #{"auto" "required"} (set (map #(-> % :decoded :tool_choice) @requests))))+ (doseq [{:keys [decoded] :as req} @requests]+ (is (= :post (:method req)))+ (is (= "http://vllm.internal:8000/v1/chat/completions" (:url req)))+ (is (= :json (:as req))+ "a probe reads a whole response; :stream would leave the verdict unreadable")+ (is (nil? (:stream decoded)))+ (is (= 0 (:temperature decoded)))+ (is (= 2048 (:max_tokens decoded))+ "must not drift from the floor a forced tool call is raised to — the preflight's promise is+ that a model which connected can already clear the budget it will be run at")+ (is (= 1 (count (:tools decoded))))+ (is (= "record_table_name" (-> decoded :tools first :function :name))))))))++(deftest preflight-surfaces-an-http-error-as-a-provider-error-test+ (testing "a 400 from /chat/completions is the server rejecting the request, not a contract failure —+ it must keep vLLM's own status text rather than being reworded as a preflight verdict"+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]+ (if (re-find #"/models$" (str url))+ {:status 200 :body {:data [{:id "vllm-test" :max_model_len 32768}]}}+ (throw (ex-info "clj-http: status 400"+ {:status 400+ :body "{\"message\":\"cannot compile grammar\"}"}))))]+ (let [e (try (vllm/list-models {:credentials credentials :probe? true})+ (catch clojure.lang.ExceptionInfo e e))]+ (is (re-find #"vLLM rejected the request" (ex-message e)))+ (is (= :provider-api-error (:error-code (ex-data e))))))))++(deftest preflight-accepts-a-catalog-without-a-context-window-test+ (testing "`max_model_len` is vLLM's own field: Ollama, LM Studio, and TGI omit it. The floor is+ best-effort, so a catalog without it connects rather than being rejected on a missing field"+ (is (= {:models [{:id "vllm-test" :display_name "vllm-test"}]+ :probed-model "vllm-test"+ :learned-config {vllm/reasoning-config-key "false"}}+ (probe! [{:id "vllm-test"}] tool-calling-message)))))++(deftest preflight-does-not-leak-the-context-window-to-the-client-test+ (testing "`max_model_len` feeds the context check and stops there — the model list the admin sees carries+ only what the dropdown renders"+ (let [models (:models (probe! [{:id "vllm-test" :max_model_len 32768 :parent nil :root "org/Model"}]+ tool-calling-message))]+ (is (= [{:id "vllm-test" :display_name "vllm-test"}] models)))))diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex e9be9153967..172ba28275d 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -74,6 +74,10 @@(#'self/parse-provider-model "moonshot/kimi-k3")))(is (=? {:provider "google" :model "google/gemini-3.5-flash" :ai-proxy? false}(#'self/parse-provider-model "google/google/gemini-3.5-flash"))))+ (testing "a vLLM served model is often a Hugging Face repo id, so the model segment keeps its slashes"+ (llm.tu/with-connections [(llm.tu/connection "vllm")]+ (is (=? {:provider "vllm" :model "mlx-community/Qwen3-14B-4bit" :ai-proxy? false}+ (#'self/parse-provider-model "vllm/mlx-community/Qwen3-14B-4bit")))))(testing "serves the managed connection through the wire family the model names"(is (=? {:provider "anthropic" :model "claude-haiku-4-5" :ai-proxy? true}(#'self/parse-provider-model "metabase/anthropic/claude-haiku-4-5")))@@ -94,7 +98,8 @@(is (fn? (#'self/resolve-adapter "zai")))(is (fn? (#'self/resolve-adapter "mistral")))(is (fn? (#'self/resolve-adapter "moonshot")))- (is (fn? (#'self/resolve-adapter "google"))))+ (is (fn? (#'self/resolve-adapter "google")))+ (is (fn? (#'self/resolve-adapter "vllm"))))(testing "throws for unknown provider"(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Unknown LLM provider"(#'self/resolve-adapter "unknown")))))@@ -969,7 +974,17 @@;; but other stuff is notfalse (RuntimeException. "oops");; a wrapped non-transient cause stays non-retryable- false (ex-info "boom" {} (IllegalArgumentException. "bad")))))+ false (ex-info "boom" {} (IllegalArgumentException. "bad"))))+ (testing ":retryable? false outranks both the status check and the cause walk"+ (are [x y] (= x (#'self/retryable-error? y))+ false (ex-info "vLLM stopped responding"+ {:api-error true :error-code :vllm-timeout :retryable? false}+ (java.net.SocketTimeoutException. "Read timed out"))+ false (ex-info "vLLM queue full" {:status 429 :retryable? false})))+ (testing "the tag is an opt-out only — :retryable? true cannot force a retry"+ (are [x y] (= x (#'self/retryable-error? y))+ true (ex-info "rate limited" {:status 429 :retryable? true})+ false (ex-info "bad request" {:status 400 :retryable? true}))))(deftest retry-delay-ms-test(testing "backoff"diff --git a/test/metabase/metabot/settings_test.clj b/test/metabase/metabot/settings_test.cljindex 9b3b1cb32ab..b53e152c817 100644--- a/test/metabase/metabot/settings_test.clj+++ b/test/metabase/metabot/settings_test.clj@@ -195,6 +195,39 @@(with-selected-model model-ref(is (= expected (metabot.settings/llm-metabot-supports-reasoning?)))))))))+(deftest metabot-supports-reasoning-vllm-test+ (testing "vLLM answers from what the connect-time probe recorded on the connection, since neither its catalog+ nor its model names carry a reasoning field"+ (doseq [recorded ["true" "false"]]+ (testing (str "recorded " recorded)+ (with-connections [(connection "vllm" "vllm" {:base-url "http://vllm.internal:8000/v1"+ :model-reasoning recorded})]+ (with-selected-model "vllm/vllm-test"+ (is (= (= "true" recorded) (metabot.settings/llm-metabot-supports-reasoning?))))))))+ (testing "an unprobed server defaults to the non-reasoning renderer rather than guessing"+ (with-connections [(connection "vllm" "vllm" {:base-url "http://vllm.internal:8000/v1"})]+ (with-selected-model "vllm/vllm-test"+ (is (false? (metabot.settings/llm-metabot-supports-reasoning?)))))))++(deftest metabot-configured-with-a-keyless-vllm-connection-test+ (testing "a vLLM server started without --api-key is a complete configuration: the base URL is the credential"+ (with-connections [(connection "vllm" "vllm" {:base-url "http://vllm.internal:8000/v1"})]+ (with-selected-model "vllm/vllm-test"+ (is (true? (metabot.settings/llm-metabot-configured?))))))+ (testing "and a connection carrying only a key is not"+ (with-connections [(connection "vllm" "vllm" {:api-key "local-dev-key"})]+ (with-selected-model "vllm/vllm-test"+ (is (false? (metabot.settings/llm-metabot-configured?)))))))++(deftest metabot-provider-keeps-slashes-in-a-vllm-model-test+ (testing "a served model named after its Hugging Face repo keeps its slashes — everything after the first+ segment is the model"+ (with-connections [(connection "vllm" "vllm" {:base-url "http://vllm.internal:8000/v1"})]+ (mt/discard-setting-changes [llm-metabot-provider]+ (metabot.settings/llm-metabot-provider! "vllm/mlx-community/Qwen3-14B-4bit")+ (is (=? {:provider "vllm" :model "mlx-community/Qwen3-14B-4bit"}+ (#'metabot.self/parse-provider-model (metabot.settings/llm-metabot-provider))))))))+;;; ------------------------------------------- validate-metabot-provider! Tests -------------------------------------------;; The validator is private; exercise it through the setting setter.