Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Support Anthropic partner models via the Google provider (#80133)

appleby5bf02e30d7ac
diff --git a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderConnectionForm.tsx b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderConnectionForm.tsx
index 1dcdc221396..17fd42b3906 100644
--- a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderConnectionForm.tsx
+++ b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderConnectionForm.tsx
@@ -53,21 +53,28 @@ export function ProviderConnectionForm({
const [config, setConfig] = useState<LlmProviderConfig>(
connection?.config ?? {},
);
- // On edit, start the picker on the model the connection is actually serving Metabot, not the type default.
+ // On edit, start the picker on the model the connection is actually serving, not the type default.
const modelRef = useSetting("llm-metabot-provider");
const [model, setModel] = useState<string | undefined>(() => {
const type = providerTypes.find(
(option) => option.type === connection?.type,
);
+ const isOnCatalog = (id?: string | null): id is string =>
+ type?.models.some((typeModel) => typeModel.id === id) ?? false;
const [refKey, ...refModelParts] = (modelRef ?? "").split("/");
const refModel = refModelParts.join("/");
if (
connection != null &&
refKey === connection.key &&
- type?.models.some((typeModel) => typeModel.id === refModel)
+ isOnCatalog(refModel)
) {
return refModel;
}
+ // Metabot points elsewhere: fall back to the model this connection was last verified against.
+ const probedModel = connection?.config?.["probed-model"];
+ if (isOnCatalog(probedModel)) {
+ return probedModel;
+ }
return type?.default_model ?? undefined;
});
const [error, setError] = useState<string | undefined>();
diff --git a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderConnectionForm.unit.spec.tsx b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderConnectionForm.unit.spec.tsx
index 24d3eae2248..c268ac96da9 100644
--- a/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderConnectionForm.unit.spec.tsx
+++ b/frontend/src/metabase/metabot/components/AIProviderConfigurationForm/ProviderConnectionForm.unit.spec.tsx
@@ -124,8 +124,12 @@ const GOOGLE_TYPE = createMockLlmProviderType({
label: "Google Gemini Enterprise",
default_model: "google/gemini-3.5-flash",
models: [
- { id: "google/gemini-3.5-flash", display_name: "gemini-3.5-flash" },
- { id: "google/gemini-3.6-flash", display_name: "gemini-3.6-flash" },
+ { id: "google/gemini-3.5-flash", display_name: "Gemini 3.5 Flash" },
+ { id: "google/gemini-3.6-flash", display_name: "Gemini 3.6 Flash" },
+ {
+ id: "anthropic/claude-sonnet-4-6",
+ display_name: "Claude Sonnet 4.6",
+ },
],
required_any: [["service-account-key"], ["oauth-access-token", "project-id"]],
fields: [
@@ -312,7 +316,7 @@ describe("ProviderConnectionForm with fields behind a choice", () => {
await pickGoogle();
await userEvent.click(screen.getByLabelText("Model"));
await userEvent.click(
- await screen.findByRole("option", { name: "gemini-3.6-flash" }),
+ await screen.findByRole("option", { name: "Gemini 3.6 Flash" }),
);
await userEvent.upload(
screen.getByLabelText("Service account key file file input"),
@@ -353,7 +357,67 @@ describe("ProviderConnectionForm editing a fixed-catalog connection", () => {
);
expect(await screen.findByLabelText("Model")).toHaveValue(
- "gemini-3.6-flash",
+ "Gemini 3.6 Flash",
+ );
+ });
+
+ it("starts the model picker on the probed model when Metabot points at another connection", async () => {
+ setupGoogle(
+ createMockLlmProviderConnection({
+ key: "google",
+ type: "google",
+ name: "Google Gemini Enterprise",
+ config: {
+ "auth-method": "oauth-token",
+ "oauth-access-token": "**********en",
+ "project-id": "my-project",
+ "probed-model": "anthropic/claude-sonnet-4-6",
+ },
+ }),
+ { modelRef: "anthropic/claude-sonnet-4-6" },
+ );
+
+ expect(await screen.findByLabelText("Model")).toHaveValue(
+ "Claude Sonnet 4.6",
+ );
+ });
+
+ it("starts the model picker on the type default when there is no probed model", async () => {
+ setupGoogle(
+ createMockLlmProviderConnection({
+ key: "google",
+ type: "google",
+ name: "Google Gemini Enterprise",
+ config: {
+ "auth-method": "oauth-token",
+ "oauth-access-token": "**********en",
+ "project-id": "my-project",
+ },
+ }),
+ );
+
+ expect(await screen.findByLabelText("Model")).toHaveValue(
+ "Gemini 3.5 Flash",
+ );
+ });
+
+ it("starts the model picker on the type default when the probed model left the catalog", async () => {
+ setupGoogle(
+ createMockLlmProviderConnection({
+ key: "google",
+ type: "google",
+ name: "Google Gemini Enterprise",
+ config: {
+ "auth-method": "oauth-token",
+ "oauth-access-token": "**********en",
+ "project-id": "my-project",
+ "probed-model": "google/gemini-2.0-flash",
+ },
+ }),
+ );
+
+ expect(await screen.findByLabelText("Model")).toHaveValue(
+ "Gemini 3.5 Flash",
);
});
diff --git a/src/metabase/llm/api/provider.clj b/src/metabase/llm/api/provider.clj
index ab0285b93d2..a0b98fa3b15 100644
--- a/src/metabase/llm/api/provider.clj
+++ b/src/metabase/llm/api/provider.clj
@@ -140,16 +140,31 @@
status (or status status-code)]
(and api-error (number? status) (<= 400 status 499))))
+(defn- selected-model
+ "The model `llm-metabot-provider` names for `conn-key`, or nil when the selection points elsewhere.
+
+ What a connection is verified against when the client names no model: for a type that checks more than its
+ credentials, the model the connection is actually serving is a better probe target than whichever one the
+ provider happens to list first."
+ [conn-key]
+ (let [model-ref (metabot.settings/llm-metabot-provider)]
+ (when (= conn-key (llm.provider/model-ref->connection-key model-ref))
+ (llm.provider/model-ref->model model-ref))))
+
(defn- list-connection-models*
"List `conn`'s models. `config-override` stands in for the stored credentials when validating a connection that has
- not been saved yet. Returns `{:models [...]}` or `{:models [] :error msg}` for a credential-level failure.
+ not been saved yet. Returns `{:models [...]}` or `{:models [...] :error msg}` for a client-level failure.
The managed provider is answered from the registry — it serves one model through the proxy, there is nothing to
fetch, and calling out would fail on instances that cannot reach it.
A type whose catalog is fixed but whose credentials are its own (Google, whose listing endpoint reports models
that are not really available) still makes the call, because it is what verifies those credentials; the catalog
- comes from the registry and the call is made against its first model.
+ comes from the registry, and the call is made against the model this connection is known to serve.
+
+ `:model` is a request the type must honour or fail. `:proposed-model` is only our guess at what this connection
+ serves — the `:probed-model` an earlier probe recorded, or the catalog's first entry — which a type is free to
+ ignore.
A type that names its model in `:config` (Azure, whose deployments its listing endpoint does not return) makes
the call for the same reason, but the model it serves comes from the connection rather than from the empty list
@@ -157,27 +172,33 @@
`probe?` asks a type that can check more than its credentials to do so — vLLM exercises the tool calling and
structured output the agent loop depends on against the model it will run on. Only [[verify-credentials!]] sets
- it: a probe generates, so it is far too slow for a plain listing. A probe reports back the model it exercised as
- `:probed-model` and anything it determined about the connection as `:learned-config`, both passed through here."
- [{:keys [type config]} config-override model probe?]
+ it: a probe generates, so it is far too slow for a plain listing. A probe reports whatever it determined about the
+ connection as `:learned-config`, passed through here for [[verify-credentials!]]'s callers to store on it."
+ [{conn-key :key :keys [type config]} config-override model probe?]
(let [fixed (llm.provider/fixed-models type)]
(if (llm.provider/managed-type? type)
{:models (vec fixed)}
(let [config (llm.provider/with-field-defaults type (or config-override config))
configured-model (llm.provider/connection-model type config)
- model (or model configured-model (:id (first fixed)))]
+ ;; Our best guess at the model to try if caller did not specify a model.
+ proposed-model (or (:probed-model config) (:id (first fixed)))
+ model (or model configured-model (selected-model conn-key))
+ config-models (cond
+ configured-model [{:id configured-model
+ :display_name (last (str/split configured-model #"/"))}]
+ fixed (vec fixed))]
(try
(let [listed (metabot.self/list-models type (cond-> {:credentials config}
- model (assoc :model model)
- probe? (assoc :probe? true)))]
- (merge (select-keys listed [:probed-model :learned-config])
- {:models (cond
- configured-model [{:id configured-model :display_name (last (str/split configured-model #"/"))}]
- fixed (vec fixed)
- :else (vec (:models listed)))}))
+ model (assoc :model model)
+ proposed-model (assoc :proposed-model proposed-model)
+ probe? (assoc :probe? true)))]
+ (merge (select-keys listed [:learned-config])
+ {:models (or config-models (vec (:models listed)))}))
(catch clojure.lang.ExceptionInfo e
(if (provider-client-error? e)
- {:models [] :error (.getMessage e)}
+ ;; Keep offering config-models, otherwise admin has no way to select a different model to fix "model not
+ ;; served in given region" errors.
+ {:models (or config-models []) :error (.getMessage e)}
(throw e))))))))
(def ^:private models-cache-ttl-ms
@@ -191,9 +212,13 @@
(defn- models-cache-key
"What a cached model list is filed under. The config is reduced to a hash rather than held as-is: it carries the
connection's API key, and a cache entry outlives the connection that produced it. Hashing it also retires the
- entry the moment a credential is rotated, instead of serving the old list until the TTL runs out."
+ entry the moment a credential is rotated, instead of serving the old list until the TTL runs out.
+
+ The selected model is part of the key because it is what a listing is verified against when the client names no
+ model. Repointing Metabot at a model the connection cannot serve has to retire a cached success, and picking a
+ model that works again has to retire the cached error, rather than either standing until the TTL runs out."
[{conn-key :key :keys [type config]}]
- [conn-key type (hash config)])
+ [conn-key type (hash config) (selected-model conn-key)])
(defn- connection-models-response
[{conn-key :key conn-name :name :keys [type] :as conn}]
@@ -235,32 +260,21 @@
for a type that probes more than its credentials, by exercising the model it will run on. Throws a 400 carrying
the provider's own message when the credentials are rejected.
- Returns `{:probed-model :learned-config}`: the model a probe exercised, for a type with no default model to fall
- back on, and whatever the probe determined about the connection, for the caller to store on it.
-
- The listing that verified the credentials is seeded into [[models-cache]] under the connection as it will be
- stored, so the model refetch the client fires right after saving is answered from it instead of round-tripping
- to the provider a second time."
+ Returns the model listing and `:learned-config`: whatever the probe determined about the connection, for the
+ caller to store on it. A probe records the model it exercised as `:probed-model`."
[conn config model]
(when-not (llm.provider/managed-type? (:type conn))
- (let [{:keys [error learned-config] :as listed} (list-connection-models* conn config model true)]
+ (let [{:keys [error] :as listed} (list-connection-models* conn config model true)]
(when error
(throw (ex-info error {:status-code 400 :api-error true})))
- (swap! models-cache cache/miss
- (models-cache-key (assoc conn :config (merge config learned-config)))
- (select-keys listed [:models]))
- (select-keys listed [:probed-model :learned-config]))))
-
-(defn- selected-model
- "The model `llm-metabot-provider` names for `conn-key`, or nil when the selection points elsewhere.
+ listed)))
- What an edited connection is verified against when the client names no model: for a type that checks more than
- its credentials, the model the connection is actually serving is a better probe target than whichever one the
- provider happens to list first."
- [conn-key]
- (let [model-ref (metabot.settings/llm-metabot-provider)]
- (when (= conn-key (llm.provider/model-ref->connection-key model-ref))
- (llm.provider/model-ref->model model-ref))))
+(defn- seed-models-cache!
+ "Cache the listing that verified `conn` under its post-save config and selected model. Call this only after any
+ save-triggered repointing, so the model refetch the client fires right after saving uses the same cache key."
+ [conn listed]
+ (when listed
+ (swap! models-cache cache/miss (models-cache-key conn) (select-keys listed [:models]))))
(defn- connection-model-ref
"The `connection-key/model` reference that points Metabot at `conn`: the model the connection's own config names
@@ -408,14 +422,15 @@
:name (or (not-empty name) (str (:label provider-type)))
:config config}]
(llm.provider/validate-config! type config)
- (let [{:keys [probed-model learned-config]} (verify-credentials! conn config model)
+ (let [{:keys [learned-config] :as listed} (verify-credentials! conn config model)
conn (update conn :config merge learned-config)
had-usable-model? (metabot-has-a-usable-model?)]
(llm.provider/set-connections! (conj (llm.provider/stored-connections) conn))
(when-not had-usable-model?
;; a type with no default model — vLLM, which serves whatever the operator loaded — starts on the model
;; the probe exercised, so connecting one leaves the instance working rather than model-less
- (select-model-for-new-connection! conn (or model probed-model)))
+ (select-model-for-new-connection! conn (or model (:probed-model learned-config))))
+ (seed-models-cache! conn listed)
(connection-response (assoc conn :source :db))))))
(api.macros/defendpoint :put "/providers/:key"
@@ -448,11 +463,13 @@
;; what the connection will actually run on: the stored config with the environment layered back over it
effective (merge (:config merged) env-config)]
(llm.provider/validate-config! (:type merged) effective)
- (let [{:keys [learned-config]} (verify-credentials! merged effective (or model (selected-model conn-key)))
+ (let [{:keys [learned-config] :as listed}
+ (verify-credentials! merged effective (or model (selected-model conn-key)))
merged (update merged :config merge learned-config)
effective (merge effective learned-config)]
(llm.provider/set-connections! (assoc stored idx merged))
(follow-edited-connection-model! (assoc merged :config effective) model)
+ (seed-models-cache! (assoc merged :config effective) listed)
(connection-response (assoc (merge live merged) :config effective)))))
(api.macros/defendpoint :delete "/providers/:key" :- :nil
diff --git a/src/metabase/llm/provider.clj b/src/metabase/llm/provider.clj
index 5a6a81ac05b..6b95bf472b4 100644
--- a/src/metabase/llm/provider.clj
+++ b/src/metabase/llm/provider.clj
@@ -21,7 +21,8 @@
[metabase.llm.settings :as llm.settings]
[metabase.settings.core :as setting]
[metabase.util :as u]
- [metabase.util.i18n :refer [deferred-tru tru]]))
+ [metabase.util.i18n :refer [deferred-tru tru]]
+ [metabase.util.log :as log]))
(set! *warn-on-reflection* true)
@@ -171,11 +172,17 @@
:default-model "google/gemini-3.5-flash"
;; The Gemini Enterprise Agent Platform has no listing endpoint we can trust — the one it exposes reports models
;; that are not really available and omits ones that are — so the models Metabot is known to work with are fixed
- ;; here, and connecting validates the credentials against the model chosen in the connection form with a free
- ;; `countTokens` probe. Which of them a project can actually reach depends on its location.
- :models [{:id "google/gemini-3.5-flash" :display_name "gemini-3.5-flash"}
- {:id "google/gemini-3.6-flash" :display_name "gemini-3.6-flash"}
- {:id "google/gemini-3.7-flash" :display_name "gemini-3.7-flash"}]
+ ;; here, and connecting validates the credentials against the model chosen in the connection form with a probe.
+ ;; Which of them a project can actually reach depends on its location.
+ :models [{:id "google/gemini-3.5-flash" :display_name "Gemini 3.5 Flash"}
+ {:id "google/gemini-3.6-flash" :display_name "Gemini 3.6 Flash"}
+ {:id "google/gemini-3.7-flash" :display_name "Gemini 3.7 Flash"}
+ {:id "anthropic/claude-fable-5" :display_name "Claude Fable 5"}
+ {:id "anthropic/claude-opus-5" :display_name "Claude Opus 5"}
+ {:id "anthropic/claude-opus-4-6" :display_name "Claude Opus 4.6"}
+ {:id "anthropic/claude-sonnet-5" :display_name "Claude Sonnet 5"}
+ {:id "anthropic/claude-sonnet-4-6" :display_name "Claude Sonnet 4.6"}
+ {:id "anthropic/claude-haiku-4-5@20251001" :display_name "Claude Haiku 4.5"}]
;; A service account key authenticates on its own (it can carry the project); an OAuth token needs the project
;; named beside it.
:required-any [[:service-account-key] [:oauth-access-token :project-id]]
@@ -789,18 +796,23 @@
stored (stored-connections)
idx (first (keep-indexed (fn [i conn] (when (= conn-key (:key conn)) i)) stored))]
;; a client echoing back the mask [[metabase.settings.core/obfuscate-value]] handed it is not entering a new
- ;; value, the same way [[metabase.settings.core/set!]] treats sensitive settings
- (when-not (setting/obfuscated-value? value)
- (when value
- (validate-config-field! group-type field {field value}))
- (cond
- idx (set-connections! (update-in stored [idx :config]
- (fn [config]
- (if value (assoc config field value) (dissoc config field)))))
- value (set-connections! (conj stored {:key conn-key
- :type group-type
- :name (str (:label (provider-type group-type)))
- :config {field value}}))))))
+ ;; value, the same way [[metabase.settings.core/set!]] treats sensitive settings. Both forms are checked: the
+ ;; mask of a newline-terminated secret (a JSON key file) matches only untrimmed, while a mask that picked up
+ ;; padding in transit matches only trimmed
+ (if (or (setting/obfuscated-value? new-value)
+ (setting/obfuscated-value? value))
+ (log/infof "Attempted to set %s to an obfuscated value. Ignoring change." (name setting-kw))
+ (do
+ (when value
+ (validate-config-field! group-type field {field value}))
+ (cond
+ idx (set-connections! (update-in stored [idx :config]
+ (fn [config]
+ (if value (assoc config field value) (dissoc config field)))))
+ value (set-connections! (conj stored {:key conn-key
+ :type group-type
+ :name (str (:label (provider-type group-type)))
+ :config {field value}})))))))
;;; -------------------------------------------------- Redaction ----------------------------------------------------
diff --git a/src/metabase/metabot/self/google.clj b/src/metabase/metabot/self/google.clj
index 00c6507fddf..b9d8be41a8f 100644
--- a/src/metabase/metabot/self/google.clj
+++ b/src/metabase/metabot/self/google.clj
@@ -2,11 +2,13 @@
"Google Gemini Enterprise Agent Platform (formerly Vertex AI) provider.
This namespace handles credentials, endpoint URLs, HTTP calls, error translation, and connect-time model validation.
- Wire-format translation is in [[metabase.metabot.self.google.stream-generate-content]].
+ Wire-format translation is in [[metabase.metabot.self.google.stream-generate-content]]
+ and [[metabase.metabot.self.google.raw-predict]].
Like openrouter and azure, models for this provider must specify the sub-provider in the `llm-metabot-provider`
- setting. For example `MB_LLM_METABOT_PROVIDER=google/google/gemini-3.6-flash`. Currently only google/gemini models
- are supported, but this may expand in the future to anthropic or other third-party models.
+ setting. For example `MB_LLM_METABOT_PROVIDER=google/google/gemini-3.6-flash`. Gemini models are served by
+ `streamGenerateContent`; Anthropic partner models (e.g. `google/anthropic/claude-sonnet-4-6`) are served by
+ `streamRawPredict`, whose payload is Anthropic's Messages API.
Credentials can be supplied via either a service account key JSON or an OAuth access token.
@@ -29,8 +31,8 @@
[metabase.llm.settings :as llm]
[metabase.metabot.self.core :as core]
[metabase.metabot.self.debug :as debug]
+ [metabase.metabot.self.google.raw-predict :as raw-predict]
[metabase.metabot.self.google.stream-generate-content :as stream-generate-content]
- [metabase.metabot.settings :as metabot.settings]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
@@ -49,21 +51,6 @@
"The model to use when the request does not name one."
"google/gemini-3.5-flash")
-(def ^:private model-context-windows
- "Input context windows for known Google Gemini models, keyed by publisher-qualified model id.
- Values:
- - https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-5-flash
- - https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-6-flash
- - https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-7-flash"
- {"google/gemini-3.5-flash" 1048576
- "google/gemini-3.6-flash" 1048576
- "google/gemini-3.7-flash" 1048576})
-
-(defn context-window-tokens
- "The input context window for `model`, or nil when it isn't one we know."
- [model]
- (get model-context-windows model))
-
;;; Auth / HTTP plumbing
(def ^:private cloud-platform-scope
@@ -256,21 +243,100 @@
(<= (count segment) max-model-segment-length)
(re-matches pattern segment))))
+(defn- model-publisher
+ "The publisher segment of a publisher-qualified model ID, e.g. `google` or `anthropic`."
+ [model]
+ (llm.provider/model-ref->connection-key model))
+
+(defn- model-id
+ "The model segment of a publisher-qualified model ID, or nil when there is none."
+ [model]
+ (llm.provider/model-ref->model model))
+
+(def ^:private model-families
+ "Map from supported model publishers to their API format."
+ {"google" :google
+ "anthropic" :anthropic})
+
+(def model-publishers
+ "The publishers whose models this adapter serves."
+ (set (keys model-families)))
+
+(defn- unqualified-model-ex
+ "The error for a `model` that does not name both a publisher and a model."
+ [model]
+ (ex-info (tru "Invalid Google model {0} — expected a publisher-qualified ID like \"google/gemini-3.5-flash\""
+ (pr-str model))
+ {:api-error true
+ :status-code 400
+ :error-code :invalid-model}))
+
+(defn- model->family
+ "Return the model family for the given `model`.
+ Throws for a publisher this adapter cannot speak to."
+ [model]
+ (or (model-families (model-publisher model))
+ (throw (if (str/blank? (model-id model))
+ (unqualified-model-ex model)
+ (ex-info (tru "Unsupported Google model {0}. Only google/* and anthropic/* models are supported."
+ (pr-str model))
+ {:api-error true
+ :status-code 400
+ :error-code :unsupported-model
+ :model model})))))
+
+(def ^:private raw-predict-method
+ "The verb that serves Anthropic partner models."
+ ":streamRawPredict")
+
+(def ^:private generate-content-method
+ "The verb that serves Gemini models, asking for its stream as SSE rather than a JSON array."
+ ":streamGenerateContent?alt=sse")
+
+(def ^:private gemini-context-windows
+ "Input context windows for known Google Gemini models, keyed by publisher-qualified model id.
+ Values:
+ - https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-5-flash
+ - https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-6-flash
+ - https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/3-7-flash"
+ {"google/gemini-3.5-flash" 1048576
+ "google/gemini-3.6-flash" 1048576
+ "google/gemini-3.7-flash" 1048576})
+
+(defn reasoning-model?
+ "Whether a publisher-qualified `model` streams its reasoning back to us.
+
+ Answers false for a model this adapter cannot serve rather than throwing the way [[model->family]] does: the
+ `llm-metabot-supports-reasoning?` setting reads this, and a provider setting Metabot cannot use must not take the
+ public settings endpoint down with it. The request path rejects the same model soon enough."
+ [model]
+ (case (model-families (model-publisher model))
+ :anthropic (raw-predict/reasoning-model? (model-id model))
+ :google (stream-generate-content/reasoning-model? (model-id model))
+ false))
+
+(defn context-window-tokens
+ "The input context window for a publisher-qualified `model`, or nil when it isn't one we know.
+
+ Answers nil for a model this adapter cannot serve rather than throwing the way [[model->family]] does, for the
+ same reason [[reasoning-model?]] does."
+ [model]
+ (case (model-families (model-publisher model))
+ :anthropic (raw-predict/context-window-tokens (model-id model))
+ :google (get gemini-context-windows model)
+ nil))
+
(defn- model-resource-path
"Returns the URL path to a publisher model resource, without the `:method` verb at the end.
The `model` must include its `{publisher}/{model}` qualifier, e.g. `google/gemini-3.5-flash`.
Both segments become path segments of the request URL, so a character that does not belong in one is rejected here.
- The `llm-metabot-provider` setting takes the Google model as free text — unlike a whitelisted provider, nothing
- upstream of this constrains it."
+ [[model->family]] has already settled the publisher by this point; the model ID is still free text."
[credentials model]
- (let [[publisher model-id] (str/split (str model) #"/" 2)]
+ (let [publisher (model-publisher model)
+ model-id (model-id model)]
(when (str/blank? model-id)
- (throw (ex-info (tru "Invalid Google model {0} — expected a publisher-qualified ID like \"google/gemini-3.5-flash\""
- (pr-str model))
- {:api-error true
- :status-code 400
- :error-code :invalid-model})))
+ (throw (unqualified-model-ex model)))
(when-not (and (valid-model-segment? publisher-pattern publisher)
(valid-model-segment? model-id-pattern model-id))
(throw (ex-info (tru (str "Invalid Google model {0} — a publisher and model ID can hold only letters, digits, "
@@ -309,23 +375,27 @@
;; the API is not available in this location.
(contains? #{404 501} status))
+(defn- google-res->msg
+ "The `res->message` callback for [[core/rethrow-api-error!]] and [[core/reducible-with-api-errors]]."
+ [credentials]
+ (let [endpoint (delay (try (api-base-url credentials) (catch Exception _ nil)))]
+ (fn [res]
+ (cond-> (google-error-msg res)
+ (and (include-endpoint-in-msg? (:status res)) @endpoint)
+ (str " " (tru "(endpoint: {0})" @endpoint))))))
+
(defn- rethrow-google-api-error!
"Rethrows a Google HTTP exception like [[core/rethrow-api-error!]], with two changes:
- For a status that satisfies [[include-endpoint-in-msg?]], the message includes the endpoint URL.
- For 404s include a hint to check that the provided location is correct."
[credentials e]
- (let [data (ex-data e)
- location (:location credentials)
- known? (conj multi-region-locations global-location)
- endpoint (delay (try (api-base-url credentials) (catch Exception _ nil)))
- res->msg (fn [res]
- (cond-> (google-error-msg res)
- (and (include-endpoint-in-msg? (:status res)) @endpoint)
- (str " " (tru "(endpoint: {0})" @endpoint))))]
+ (let [data (ex-data e)
+ location (:location credentials)
+ known? (conj multi-region-locations global-location)]
(core/rethrow-api-error!
"google"
- res->msg
+ (google-res->msg credentials)
(if (and location
(= 404 (:status data))
(not (known? location))
@@ -343,61 +413,111 @@
"The smallest `countTokens` request body for the connect-time probe."
{:contents [{:role "user" :parts [{:text "hi"}]}]})
-(defn- count-tokens!
- "Makes a free `countTokens` call for `model` and discards the response — the connect-time probe.
+(defn- validate-google-surface!
+ "Validate `credentials` and `model` for a google model.
- https://docs.cloud.google.com/gemini-enterprise-agent-platform/reference/rest/v1/projects.locations.publishers.models/countTokens
+ Round-trip a Gemini model's `countTokens` route, which is free and names the model in the URL, so a 2xx proves the
+ credential, the project, the location, and the model all resolve.
- This one call shows that the credential, the project ID, the endpoint, and the model are correct, and it uses no
- inference tokens. It agrees with real inference about model availability."
+ https://docs.cloud.google.com/gemini-enterprise-agent-platform/reference/rest/v1/projects.locations.publishers.models/countTokens"
[auth credentials model]
(core/request auth
{:method :post
:url (str (model-resource-path credentials model) ":countTokens")
:headers {"Content-Type" "application/json"}
- :body (json/encode count-tokens-probe-body)})
- nil)
+ :body (json/encode count-tokens-probe-body)}))
+
+(defn- anthropic-error-body?
+ "Whether an error `body` shape matches an Anthropic error rather than Google's.
+
+ Anthropic: `{\"type\": \"error\", \"error\": {...}}`
+ Google: `{\"error\": {\"code\": ..., \"status\": ...}}`"
+ [body]
+ (= "error" (:type (try (json/decode+kw body) (catch Exception _ nil)))))
+
+(defn- validate-anthropic-surface!
+ "Validate `credentials` and `model` for an Anthropic model.
+
+ Round-trip an Anthropic partner model's `streamRawPredict` route with an empty body in order to check whether the
+ provided credentials and model are valid without consuming any tokens. This is similar to the approach taken by the
+ azure provider in [[metabase.metabot.self.azure/validate-anthropic-surface!]].
-(defn- configured-google-model
- "The publisher-qualified model of the connection Metabot is pointed at, when that connection is a Google one."
- []
- (let [{:keys [type model]} (llm.provider/resolve-model-ref (metabot.settings/llm-metabot-provider))]
- (when (= type "google")
- model)))
+ Google resolves the credential, the project, the location, and the model from the URL before it hands the body to
+ Anthropic, so a reply matching Anthropic's error shape proves all four while spending no tokens. Anything Google
+ answers itself is rethrown: a 404 for a model this project or location cannot reach, a 403 for a publisher whose
+ data-sharing terms are not accepted, a 401 for a stale credential.
+
+ We do not use Anthropic's count-tokens endpoint here because, unlike the corresponding :countTokens for Gemini
+ models, Anthropic's count-tokens will accept any valid anthropic model, even ones that are not actually available in
+ the given location.
+
+ Unlike [[metabase.metabot.self.azure/validate-anthropic-surface!]], the status code alone cannot settle it. Google
+ rejects a model that the location does not serve with a `400 FAILED_PRECONDITION` of its own — the same status
+ Anthropic uses for the validation error that means the model *is* servable."
+ [auth credentials model]
+ (try
+ (core/request auth
+ {:method :post
+ :url (str (model-resource-path credentials model) raw-predict-method)
+ :headers {"Content-Type" "application/json"}
+ :body "{}"})
+ (catch Exception e
+ (let [{:keys [status body]} (ex-data e)]
+ (when-not (and (= 400 status) (anthropic-error-body? body))
+ (throw e))))))
+
+(defn- validate-model!
+ "Validates `model` against the surface that serves it, and discards the response."
+ [auth credentials model]
+ (case (model->family model)
+ :anthropic (validate-anthropic-surface! auth credentials model)
+ :google (validate-google-surface! auth credentials model))
+ nil)
(defn list-models
- "Validates the Google credentials and the candidate model with a free `countTokens` call.
+ "Validates the Google credentials and the candidate model with a probe, and returns an empty model list.
Similar to the Azure provider, there is no list-models call that we can use to whitelist models for the Gemini
Enterprise Agent Platform. An endpoint does exist, but it sometimes returns models that are not really available and
sometimes omits models that are available, hence we can't rely on it. See
https://github.com/googleapis/python-genai/issues/679
- We therefore take an approach similar to the Azure provider: validate credentials and caller-provided model strings
- by sending a `countTokens` probe, but always return an empty model list."
+ `:probe?` reports the model the probe verified as `:learned-config` `:probed-model`, for the connect and edit paths
+ to record on the connection and re-verify against later passing it in as the `proposed-model` on future attempts."
([] (list-models {}))
- ([{:keys [credentials model ai-proxy?]}]
- (when-let [model (or (not-empty model) (configured-google-model))]
- (try
- (let [{:keys [auth credentials]} (resolve-google-auth credentials ai-proxy?)]
- (count-tokens! auth credentials model))
- (catch Exception e
- (rethrow-google-api-error! credentials e))))
- {:models []}))
+ ([{:keys [credentials model proposed-model ai-proxy? probe?]}]
+ (if-let [model (or (not-empty model) (not-empty proposed-model))]
+ (do
+ (try
+ (let [{:keys [auth credentials]} (resolve-google-auth credentials ai-proxy?)]
+ (validate-model! auth credentials model))
+ (catch Exception e
+ (rethrow-google-api-error! credentials e)))
+ (cond-> {:models []}
+ probe? (assoc :learned-config {:probed-model model})))
+ {:models []})))
(mu/defn google-raw
- "Makes a streaming `streamGenerateContent` request to the Gemini Enterprise Agent Platform.
+ "Makes a streaming request to the Gemini Enterprise Agent Platform.
+ Gemini models stream through `streamGenerateContent`; Anthropic partner models through `streamRawPredict`.
`:ai-proxy?` is not supported and throws when it is true."
[{:keys [model input tools credentials ai-proxy?] :as opts
:or {model default-model}} :- core/LLMRequestOpts]
- (let [req (stream-generate-content/request-body opts)]
+ (let [family (model->family model)
+ req (case family
+ :anthropic (raw-predict/request-body (model-id model) opts)
+ :google (stream-generate-content/request-body opts))
+ method (case family
+ :anthropic raw-predict-method
+ :google generate-content-method)
+ res->msg (google-res->msg credentials)]
(with-span :info {:name :metabot.google/request
:model model
:msg-count (count input)
:tool-count (count tools)}
(try
(let [{:keys [auth credentials]} (resolve-google-auth credentials ai-proxy?)
- url (str (model-resource-path credentials model) ":streamGenerateContent?alt=sse")
+ url (str (model-resource-path credentials model) method)
response (core/request auth
{:method :post
:url url
@@ -408,12 +528,19 @@
(debug/capture-stream {:provider "google"
:model model
:url url
- :request req})))
+ :request req})
+ (core/reducible-with-api-errors "google" res->msg)))
(catch Exception e
(rethrow-google-api-error! credentials e))))))
(defn google
"Call the Gemini Enterprise Agent Platform, return AISDK stream."
[& args]
- (let [raw (apply google-raw args)]
- (eduction (stream-generate-content/->aisdk-chunks-xf) raw)))
+ (let [{:keys [model] :or {model default-model}} (first args)
+ raw (apply google-raw args)]
+ ;; Keep this dispatch in sync with `google-raw`, which independently uses
+ ;; `model->family` to select the request protocol.
+ (eduction (case (model->family model)
+ :anthropic (raw-predict/->aisdk-chunks-xf)
+ :google (stream-generate-content/->aisdk-chunks-xf))
+ raw)))
diff --git a/src/metabase/metabot/self/google/raw_predict.clj b/src/metabase/metabot/self/google/raw_predict.clj
new file mode 100644
index 00000000000..d769de7b57c
--- /dev/null
+++ b/src/metabase/metabot/self/google/raw_predict.clj
@@ -0,0 +1,51 @@
+(ns metabase.metabot.self.google.raw-predict
+ "Wire-format for Anthropic partner models on the Gemini Enterprise Agent Platform.
+
+ Anthropic models are served by the `streamRawPredict` method, whose payload is Anthropic's Messages with two
+ platform differences: the request URL names the model instead of the body, and `anthropic_version` is pinned to
+ `vertex-2023-10-16` and sent in the body instead of a header. Everything else (system blocks, tools, forced
+ structured output, prompt caching, streamed events) is exactly the Messages API, so this namespace is a thin wrapper
+ over [[metabase.metabot.self.claude]].
+
+ https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/partner-models/claude/use-claude"
+ (:require
+ [clojure.string :as str]
+ [metabase.metabot.self.claude :as claude]
+ [metabase.metabot.self.core :as core]
+ [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private anthropic-version
+ "Google-specific `anthropic_version`."
+ "vertex-2023-10-16")
+
+(defn- direct-api-model-id
+ "The direct-API spelling of a platform model ID.
+ The Gemini Agent Platform versions a dated model as `{model}@{date}` where the Messages API spells it
+ `{model}-{date}`. The Claude adapter's model knowledge (max_tokens ceilings, thinking support) is keyed by the
+ direct spelling."
+ [model-id]
+ (str/replace (str model-id) "@" "-"))
+
+(mu/defn request-body
+ "Build the streamRawPredict request body for an LLM request on `model-id`."
+ [model-id opts :- core/LLMRequestOpts]
+ (-> (claude/claude-request-body (assoc opts :model (direct-api-model-id model-id)))
+ (dissoc :model)
+ (assoc :anthropic_version anthropic-version)))
+
+(defn ->aisdk-chunks-xf
+ "Translates streamed Messages API events into AI SDK v5 chunks."
+ []
+ (claude/claude->aisdk-chunks-xf))
+
+(defn reasoning-model?
+ "Whether `model-id` (the model without its publisher qualifier) streams its reasoning back to us."
+ [model-id]
+ (claude/reasoning-model? (direct-api-model-id model-id)))
+
+(defn context-window-tokens
+ "The input context window for `model-id`, or nil when it isn't one we know."
+ [model-id]
+ (claude/context-window-tokens (direct-api-model-id model-id)))
diff --git a/src/metabase/metabot/self/google/stream_generate_content.clj b/src/metabase/metabot/self/google/stream_generate_content.clj
index 245540181cb..4b68649fc0c 100644
--- a/src/metabase/metabot/self/google/stream_generate_content.clj
+++ b/src/metabase/metabot/self/google/stream_generate_content.clj
@@ -184,6 +184,13 @@
(when-let [detail (finish-reason-details reason)]
(str ": " detail)))))
+(defn reasoning-model?
+ "Whether `model-id` streams its reasoning back to us.
+ No Gemini model does today: thinking arrives as parts marked `:thought`, which [[->aisdk-chunks-xf]] drops. Should
+ the adapter start forwarding them, this is where the models that send them get named."
+ [_model-id]
+ false)
+
(defn ->aisdk-chunks-xf
"Translates `streamGenerateContent` SSE events into AI SDK v5 protocol chunks.
diff --git a/src/metabase/metabot/self/vllm.clj b/src/metabase/metabot/self/vllm.clj
index c92f00a3113..ece8f14176b 100644
--- a/src/metabase/metabot/self/vllm.clj
+++ b/src/metabase/metabot/self/vllm.clj
@@ -379,21 +379,25 @@
"List the models the connection's vLLM server is serving. Pass-through: there is nothing to
whitelist, and `display_name` falls back to the served id.
- `:probe?` additionally runs [[preflight!]], and reports the model it exercised as `:probed-model`
- for the connect path to adopt, along with the `:learned-config` the probe determined. Reserved for
- the connect and edit paths — a tool-call probe on every model listing would stall the admin picker
- behind a full prefill."
+ `:probe?` additionally runs [[preflight!]], and reports what it determined as `:learned-config`,
+ for the connect path to store on the connection: whether the model reasons, and the model it
+ exercised, which the connect path adopts as the one to run on. Reserved for the connect and edit
+ paths — a tool-call probe on every model listing would stall the admin picker behind a full
+ prefill. On edit, a `:proposed-model` is re-probed only while the server still advertises it;
+ otherwise the normal candidate selection chooses a replacement."
([] (list-models {}))
- ([{:keys [credentials ai-proxy? model probe?]}]
- (let [auth (vllm-auth credentials ai-proxy?)
- entries (list-all-models auth)
- probed (when probe?
- (preflight! auth entries model))]
+ ([{:keys [credentials ai-proxy? model proposed-model probe?]}]
+ (let [auth (vllm-auth credentials ai-proxy?)
+ entries (list-all-models auth)
+ proposed (when (some #(= proposed-model (:id %)) entries)
+ proposed-model)
+ probed (when probe?
+ (preflight! auth entries (or model proposed)))]
(cond-> {:models (mapv (fn [{:keys [id] :as entry}]
{:id id :display_name (or (:name entry) id)})
entries)}
- probed (assoc :probed-model (:model probed)
- :learned-config {reasoning-config-key (str (:reasoning? probed))})))))
+ probed (assoc :learned-config {reasoning-config-key (str (:reasoning? probed))
+ :probed-model (:model probed)})))))
;;; --------------------------------------------------- Requests -------------------------------------------------
diff --git a/src/metabase/metabot/settings.clj b/src/metabase/metabot/settings.clj
index 2f9baf3dd7e..7f2b710aaf5 100644
--- a/src/metabase/metabot/settings.clj
+++ b/src/metabase/metabot/settings.clj
@@ -5,6 +5,7 @@
[metabase.llm.settings :as llm.settings]
[metabase.metabot.self.claude :as claude]
[metabase.metabot.self.deepseek :as deepseek]
+ [metabase.metabot.self.google :as google]
[metabase.metabot.self.openai :as openai]
[metabase.metabot.self.vllm :as vllm]
[metabase.settings.core :as setting :refer [defsetting]]
@@ -128,6 +129,21 @@
{:status-code 400
:value value})))))
+(defn- validate-google-model!
+ "Validate the model segment of a google connection's `{publisher}/{model-id}` model.
+ A publisher this provider serves followed by a non-blank model ID without slashes (the ID is one path segment of the
+ request URL). Throws on invalid input."
+ [value model]
+ (let [[publisher model-id] (str/split (str model) #"/" 2)]
+ (when-not (and (contains? google/model-publishers publisher)
+ (not (str/blank? model-id))
+ (not (str/includes? model-id "/")))
+ (throw (ex-info (tru "Invalid Google model {0}. Expected format: <connection>/<publisher>/<model> where <publisher> is one of: {1}"
+ (pr-str value)
+ (str/join ", " (sort google/model-publishers)))
+ {:status-code 400
+ :value value})))))
+
(defn- validate-managed-model!
"Check `model` against the fixed catalog the Metabase AI proxy serves (see the `metabase` entry in
[[metabase.llm.provider/provider-types]])."
@@ -158,6 +174,7 @@
{:status-code 400 :value value})))
(case (:type (llm.provider/connection (llm.provider/model-ref->connection-key value)))
"azure" (validate-azure-model! value model)
+ "google" (validate-google-model! value model)
"metabase" (validate-managed-model! model)
nil)))
@@ -234,6 +251,7 @@
"anthropic" (claude/reasoning-model? model)
"deepseek" (deepseek/reasoning-model? model)
"openai" (openai/reasoning-model? model)
+ "google" (google/reasoning-model? model)
"vllm" (vllm/reasoning-connection? credentials)
false)))
diff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.clj
index 043e4fd902d..a3396d73626 100644
--- a/src/metabase/settings/models/setting.clj
+++ b/src/metabase/settings/models/setting.clj
@@ -790,10 +790,12 @@
(defn obfuscated-value?
"Whether `v` looks like a value already obfuscated by [[obfuscate-value]], i.e. the client echoed back a masked
- value rather than entering a new one."
+ value rather than entering a new one.
+
+ `(?s)` so the trailing `.` can also match newlines, e.g. JSON file contents that end with a newline."
[v]
(when (seq v)
- (boolean (re-matches #"^\*{10}.{2}$" v))))
+ (boolean (re-matches #"(?s)^\*{10}.{2}$" v))))
(defn obfuscate-value
"Obfuscate the value of sensitive Setting. We'll still show the last 2 characters so admins can still check that the
diff --git a/test/metabase/llm/api/provider_test.clj b/test/metabase/llm/api/provider_test.clj
index e53a4edc94c..8e4b4a0132e 100644
--- a/test/metabase/llm/api/provider_test.clj
+++ b/test/metabase/llm/api/provider_test.clj
@@ -110,9 +110,15 @@
(is (nil? (fields "model")))
(is (= "google/gemini-3.5-flash" (:default_model google)))
(testing "and the catalog rides along so the connection form can offer the model to validate against"
- (is (= [{:id "google/gemini-3.5-flash" :display_name "gemini-3.5-flash"}
- {:id "google/gemini-3.6-flash" :display_name "gemini-3.6-flash"}
- {:id "google/gemini-3.7-flash" :display_name "gemini-3.7-flash"}]
+ (is (= [{:id "google/gemini-3.5-flash" :display_name "Gemini 3.5 Flash"}
+ {:id "google/gemini-3.6-flash" :display_name "Gemini 3.6 Flash"}
+ {:id "google/gemini-3.7-flash" :display_name "Gemini 3.7 Flash"}
+ {:id "anthropic/claude-fable-5" :display_name "Claude Fable 5"}
+ {:id "anthropic/claude-opus-5" :display_name "Claude Opus 5"}
+ {:id "anthropic/claude-opus-4-6" :display_name "Claude Opus 4.6"}
+ {:id "anthropic/claude-sonnet-5" :display_name "Claude Sonnet 5"}
+ {:id "anthropic/claude-sonnet-4-6" :display_name "Claude Sonnet 4.6"}
+ {:id "anthropic/claude-haiku-4-5@20251001" :display_name "Claude Haiku 4.5"}]
(:models google)))))
(testing "the alternative credential groups ride along so the form knows when the config is complete"
(is (= [["service-account-key"] ["oauth-access-token" "project-id"]]
@@ -247,8 +253,8 @@
(reset! opts o)
{:models [{:id "vllm-test" :display_name "vllm-test"}
{:id "other" :display_name "other"}]
- :probed-model "vllm-test"
- :learned-config {:model-reasoning "true"}})]
+ :learned-config {:model-reasoning "true"
+ :probed-model "vllm-test"}})]
(mt/with-temporary-setting-values [llm-providers []]
(mt/with-temporary-raw-setting-values [llm-metabot-provider nil]
(is (=? {:key "vllm"
@@ -263,7 +269,8 @@
(is (= "vllm/vllm-test" (metabot.settings/llm-metabot-provider)))
(testing "and what the probe learned is stored on the connection, where the request path reads it"
(is (= {:base-url "http://vllm.internal:8000/v1"
- :model-reasoning "true"}
+ :model-reasoning "true"
+ :probed-model "vllm-test"}
(stored-config "vllm")))
(is (true? (metabot.settings/llm-metabot-supports-reasoning?))))))))))
@@ -313,8 +320,8 @@
(fn [_provider o]
(reset! opts o)
{:models [{:id "served-a" :display_name "served-a"}]
- :probed-model "served-b"
- :learned-config {:model-reasoning "false"}})]
+ :learned-config {:model-reasoning "false"
+ :probed-model "served-b"}})]
(mt/with-temporary-setting-values [llm-providers [(connection "vllm" "vllm"
{:base-url "http://old.internal:8000/v1"
:model-reasoning "true"})]]
@@ -324,10 +331,57 @@
(is (=? {:model "served-b" :probe? true} @opts))
(testing "and the probe's fresh verdict replaces the one the connection was carrying"
(is (= {:base-url "http://vllm.internal:8000/v1"
- :model-reasoning "false"}
+ :model-reasoning "false"
+ :probed-model "served-b"}
(stored-config "vllm")))
(is (false? (metabot.settings/llm-metabot-supports-reasoning?))))))))))
+(deftest update-adopts-the-model-a-vllm-server-is-serving-now-test
+ (testing "vLLM still reports configured models even if no longer serving the previously :probed-model"
+ (let [opts (atom nil)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [_provider {:keys [model] :as o}]
+ (reset! opts o)
+ (when model
+ (throw (ex-info "The vLLM server is not serving Qwen/Qwen3-8B. It is serving: Qwen/Qwen3-32B."
+ {:api-error true :status-code 400})))
+ {:models [{:id "Qwen/Qwen3-32B" :display_name "Qwen/Qwen3-32B"}]
+ :learned-config {:model-reasoning "false"
+ :probed-model "Qwen/Qwen3-32B"}})]
+ (mt/with-temporary-setting-values [llm-providers [(connection "vllm" "vllm"
+ {:base-url "http://old.internal:8000/v1"
+ :model-reasoning "false"
+ :probed-model "Qwen/Qwen3-8B"})]]
+ ;; Metabot points elsewhere, so nothing but the recorded probe names a model for this connection
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider "anthropic/claude-opus-4-8"]
+ (mt/user-http-request :crowberto :put 200 "llm/providers/vllm"
+ {:config {:base-url "http://vllm.internal:8000/v1"}})
+ (testing "the recorded model is offered as a proposal the probe may decline, not as a request"
+ (is (=? {:proposed-model "Qwen/Qwen3-8B" :probe? true} @opts))
+ (is (not (contains? @opts :model))))
+ (testing "and the connection records what the server is serving now"
+ (is (= {:base-url "http://vllm.internal:8000/v1"
+ :model-reasoning "false"
+ :probed-model "Qwen/Qwen3-32B"}
+ (stored-config "vllm"))))))))))
+
+(deftest update-still-verifies-against-a-model-the-client-names-test
+ (testing "a model the admin picks is a request, and stays binding even though a recorded probe names another"
+ (let [opts (atom nil)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [_provider o]
+ (reset! opts o)
+ {:models [{:id "google/gemini-3.5-flash" :display_name "Gemini 3.5 Flash"}]
+ :learned-config {:probed-model (:model o)}})]
+ (mt/with-temporary-setting-values [llm-providers [(connection "google" "google"
+ {:oauth-access-token "ya29.token"
+ :project-id "my-project"
+ :probed-model "anthropic/claude-sonnet-4-6"})]]
+ (mt/user-http-request :crowberto :put 200 "llm/providers/google"
+ {:config {:oauth-access-token "ya29.token" :project-id "my-project"}
+ :model "google/gemini-3.7-flash"})
+ (is (= "google/gemini-3.7-flash" (:model @opts))))))))
+
(deftest create-selects-a-model-composed-from-the-config-test
(testing (str "Azure names its deployment in `:config` rather than listing models, and the form leaves a field it "
"pre-filled with the registry default out of the payload, so the default has to be filled in before "
@@ -543,6 +597,23 @@
:base-url "https://new.example.com"}})))
(is (= {:api-key "sk-ant-stored" :base-url "https://new.example.com"} (stored-config "anthropic"))))))
+(deftest update-preserves-a-masked-service-account-key-test
+ (testing (str "re-saving a Google connection without touching the key file echoes back the mask of a JSON key "
+ "that ends in a newline — the stored key has to survive it rather than be replaced by the mask")
+ (let [key-file "{\n \"type\": \"service_account\",\n \"project_id\": \"my-project\"\n}\n"]
+ (mt/with-temporary-setting-values [llm-providers [(connection "google" "google"
+ {:auth-method "service-account-key"
+ :service-account-key key-file})]]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [_provider {:keys [credentials]}]
+ (is (= key-file (:service-account-key credentials))
+ "the stored key is what gets probed, not the mask")
+ {:models []})]
+ (mt/user-http-request :crowberto :put 200 "llm/providers/google"
+ {:config {:auth-method "service-account-key"
+ :service-account-key (setting/obfuscate-value key-file)}})
+ (is (= key-file (:service-account-key (stored-config "google")))))))))
+
(deftest update-replaces-a-freshly-entered-secret-test
(mt/with-temporary-setting-values [llm-providers [(connection "anthropic" "anthropic" {:api-key "sk-ant-stored"})]]
(mt/with-dynamic-fn-redefs [metabot.self/list-models
@@ -850,24 +921,193 @@
(mt/user-http-request :crowberto :get 200 "llm/models"))
(is (= ["sk-ant-first" "sk-ant-rotated"] @keys-seen))))))
+(deftest models-are-refetched-when-the-selected-model-changes-test
+ (testing "repointing Metabot at another of a connection's models reprobes instead of reusing the cached listing"
+ (let [probed (atom [])]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models (fn [_provider {:keys [model proposed-model]}]
+ (swap! probed conj (or model proposed-model))
+ {:models []})]
+ (mt/with-temporary-setting-values [llm-providers [(connection "cache-keying-model" "google"
+ {:oauth-access-token "ya29.token"
+ :project-id "my-project"})]]
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider "cache-keying-model/google/gemini-3.5-flash"]
+ (mt/user-http-request :crowberto :get 200 "llm/models")
+ (mt/user-http-request :crowberto :get 200 "llm/models"))
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider "cache-keying-model/anthropic/claude-opus-5"]
+ (mt/user-http-request :crowberto :get 200 "llm/models")))
+ (is (= ["google/gemini-3.5-flash" "anthropic/claude-opus-5"] @probed))))))
+
+(deftest models-cache-is-seeded-under-the-post-save-selection-test
+ (testing "creating the first usable connection caches its listing under the model selected by the save"
+ (let [calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models (fn [& _]
+ (swap! calls inc)
+ {:models []})]
+ (mt/with-temporary-setting-values [llm-providers []]
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider nil]
+ (mt/user-http-request :crowberto :post 200 "llm/providers"
+ {:type "google"
+ :key "post-save-create"
+ :model "anthropic/claude-opus-5"
+ :config {:oauth-access-token "ya29.token"
+ :project-id "my-project"}})
+ (is (= "post-save-create/anthropic/claude-opus-5"
+ (metabot.settings/llm-metabot-provider)))
+ (mt/user-http-request :crowberto :get 200 "llm/models")
+ (is (= 1 @calls) "the post-save model refetch reuses the credential probe"))))))
+ (testing "editing the active fixed-catalog model caches its listing under the followed selection"
+ (let [calls (atom 0)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models (fn [& _]
+ (swap! calls inc)
+ {:models []})]
+ (mt/with-temporary-setting-values [llm-providers [(connection "post-save-update" "google"
+ {:oauth-access-token "ya29.token"
+ :project-id "my-project"})]]
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider
+ "post-save-update/google/gemini-3.5-flash"]
+ (mt/user-http-request :crowberto :put 200 "llm/providers/post-save-update"
+ {:config {}
+ :model "anthropic/claude-opus-5"})
+ (is (= "post-save-update/anthropic/claude-opus-5"
+ (metabot.settings/llm-metabot-provider)))
+ (mt/user-http-request :crowberto :get 200 "llm/models")
+ (is (= 1 @calls) "the post-save model refetch reuses the credential probe")))))))
+
(deftest models-for-a-connection-with-a-fixed-catalog-test
(testing (str "Google's models are the registry's, so every connection of the type offers both of them in the "
"model picker — but the call is still made, because it is what verifies the credentials")
(let [probed (atom nil)]
- (mt/with-dynamic-fn-redefs [metabot.self/list-models (fn [_provider {:keys [model]}]
- (reset! probed model)
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models (fn [_provider {:keys [model proposed-model]}]
+ (reset! probed (or model proposed-model))
{:models []})]
(mt/with-temporary-setting-values [llm-providers [(connection "gemini-catalog" "google"
{:oauth-access-token "ya29.token"
:project-id "my-project"})]]
- (is (= [{:key "gemini-catalog"
- :name "gemini-catalog"
- :type "google"
- :models [{:id "google/gemini-3.5-flash" :display_name "gemini-3.5-flash"}
- {:id "google/gemini-3.6-flash" :display_name "gemini-3.6-flash"}
- {:id "google/gemini-3.7-flash" :display_name "gemini-3.7-flash"}]}]
- (mt/user-http-request :crowberto :get 200 "llm/models")))
- (is (= "google/gemini-3.5-flash" @probed)))))))
+ ;; nothing names a model for this connection, so the probe has only the catalog to go on
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider nil]
+ (is (= [{:key "gemini-catalog"
+ :name "gemini-catalog"
+ :type "google"
+ :models [{:id "google/gemini-3.5-flash" :display_name "Gemini 3.5 Flash"}
+ {:id "google/gemini-3.6-flash" :display_name "Gemini 3.6 Flash"}
+ {:id "google/gemini-3.7-flash" :display_name "Gemini 3.7 Flash"}
+ {:id "anthropic/claude-fable-5" :display_name "Claude Fable 5"}
+ {:id "anthropic/claude-opus-5" :display_name "Claude Opus 5"}
+ {:id "anthropic/claude-opus-4-6" :display_name "Claude Opus 4.6"}
+ {:id "anthropic/claude-sonnet-5" :display_name "Claude Sonnet 5"}
+ {:id "anthropic/claude-sonnet-4-6" :display_name "Claude Sonnet 4.6"}
+ {:id "anthropic/claude-haiku-4-5@20251001" :display_name "Claude Haiku 4.5"}]}]
+ (mt/user-http-request :crowberto :get 200 "llm/models")))
+ (is (= "google/gemini-3.5-flash" @probed))))))))
+
+(deftest models-listing-probes-the-model-the-connection-was-verified-against-test
+ (testing (str "Google's catalog spans two families served in different locations, so probing whichever model the "
+ "registry lists first can fail on a connection that works — the model the connection was verified "
+ "against is probed instead, while the picker still offers the whole catalog")
+ (let [probed (atom nil)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models (fn [_provider {:keys [model proposed-model]}]
+ (reset! probed (or model proposed-model))
+ {:models []})]
+ (mt/with-temporary-setting-values [llm-providers [(connection "claude-only" "google"
+ {:oauth-access-token "ya29.token"
+ :project-id "my-project"
+ :location "us-east5"
+ :probed-model "anthropic/claude-sonnet-4-6"})]]
+ (is (=? [{:key "claude-only" :models [{:id "google/gemini-3.5-flash"} some? some? some? some? some? some? some? some?]}]
+ (mt/user-http-request :crowberto :get 200 "llm/models")))
+ (is (= "anthropic/claude-sonnet-4-6" @probed)))))))
+
+(deftest models-listing-probes-the-model-metabot-is-pointed-at-test
+ (testing (str "an environment-configured connection is never written back to, so it carries no probed model — the "
+ "selection Metabot runs on names it instead")
+ (let [probed (atom nil)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models (fn [_provider {:keys [model proposed-model]}]
+ (reset! probed (or model proposed-model))
+ {:models []})]
+ (mt/with-temporary-setting-values [llm-providers [(connection "env-google" "google"
+ {:oauth-access-token "ya29.token"
+ :project-id "my-project"})]]
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider "env-google/anthropic/claude-opus-5"]
+ (mt/user-http-request :crowberto :get 200 "llm/models")
+ (is (= "anthropic/claude-opus-5" @probed))))))))
+
+(deftest models-listing-prefers-the-selection-over-the-recorded-probe-test
+ (testing "a selection made since the connection was saved is fresher than what it was last verified against"
+ (let [probed (atom nil)]
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models (fn [_provider {:keys [model proposed-model]}]
+ (reset! probed (or model proposed-model))
+ {:models []})]
+ (mt/with-temporary-setting-values [llm-providers [(connection "reselected-google" "google"
+ {:oauth-access-token "ya29.token"
+ :project-id "my-project"
+ :probed-model "anthropic/claude-sonnet-4-6"})]]
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider "reselected-google/anthropic/claude-opus-5"]
+ (mt/user-http-request :crowberto :get 200 "llm/models")
+ (is (= "anthropic/claude-opus-5" @probed))))))))
+
+(deftest models-listing-keeps-the-fixed-catalog-when-the-probed-model-is-not-served-test
+ (testing (str "a selection naming a model the project cannot serve fails the probe — the catalog it was picked "
+ "from is still offered, so the admin can select a model that works instead of facing an empty picker")
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [_provider _opts]
+ (throw (ex-info "Google API error: model not found"
+ {:api-error true :status 404})))]
+ (mt/with-temporary-setting-values [llm-providers [(connection "wrong-model-google" "google"
+ {:oauth-access-token "ya29.token"
+ :project-id "my-project"})]]
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider "wrong-model-google/anthropic/claude-opus-5"]
+ (is (=? [{:key "wrong-model-google"
+ :models [{:id "google/gemini-3.5-flash"} some? some? some? some? some? some? some? some?]
+ :error "Google API error: model not found"}]
+ (mt/user-http-request :crowberto :get 200 "llm/models"))))))))
+
+(deftest models-listing-keeps-the-fixed-catalog-when-the-model-is-not-permitted-test
+ (testing (str "a 403 for a publisher whose terms the project has not accepted is about the model, not the "
+ "credentials — the catalog stays, because picking another model is the admin's way out")
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [_provider _opts]
+ (throw (ex-info "Google API error: PERMISSION_DENIED"
+ {:api-error true :status 403})))]
+ (mt/with-temporary-setting-values [llm-providers [(connection "forbidden-model-google" "google"
+ {:oauth-access-token "ya29.token"
+ :project-id "my-project"})]]
+ (mt/with-temporary-raw-setting-values [llm-metabot-provider "forbidden-model-google/anthropic/claude-opus-5"]
+ (is (=? [{:key "forbidden-model-google"
+ :models [{:id "google/gemini-3.5-flash"} some? some? some? some? some? some? some? some?]
+ :error "Google API error: PERMISSION_DENIED"}]
+ (mt/user-http-request :crowberto :get 200 "llm/models"))))))))
+
+(deftest models-listing-keeps-the-fixed-catalog-when-the-credentials-are-rejected-test
+ (testing (str "rejected credentials are reported as the error on the connection rather than implied by an empty "
+ "picker — a catalog the type owns does not depend on the call that failed")
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models
+ (fn [_provider _opts]
+ (throw (ex-info "Google API error: invalid authentication credentials"
+ {:api-error true :status 401})))]
+ (mt/with-temporary-setting-values [llm-providers [(connection "bad-key-google" "google"
+ {:oauth-access-token "ya29.expired"
+ :project-id "my-project"})]]
+ (is (=? [{:key "bad-key-google"
+ :name "bad-key-google"
+ :type "google"
+ :models [{:id "google/gemini-3.5-flash"} some? some? some? some? some? some? some? some?]
+ :error "Google API error: invalid authentication credentials"}]
+ (mt/user-http-request :crowberto :get 200 "llm/models")))))))
+
+(deftest create-records-the-model-the-probe-verified-test
+ (testing "connecting Google against a partner model records it, so the listing that follows probes it too"
+ (mt/with-dynamic-fn-redefs [metabot.self/list-models (fn [_provider {:keys [model]}]
+ {:models []
+ :learned-config {:probed-model model}})]
+ (mt/with-temporary-setting-values [llm-providers []]
+ (mt/user-http-request :crowberto :post 200 "llm/providers"
+ {:type "google"
+ :config {:oauth-access-token "ya29.token" :project-id "my-project"}
+ :model "anthropic/claude-sonnet-4-6"})
+ (is (= {:oauth-access-token "ya29.token"
+ :project-id "my-project"
+ :probed-model "anthropic/claude-sonnet-4-6"}
+ (stored-config "google")))))))
(deftest models-for-a-connection-that-names-its-own-model-test
(testing "Azure serves a deployment its listing endpoint never returns, so the connection's own model is reported"
diff --git a/test/metabase/llm/provider_test.clj b/test/metabase/llm/provider_test.clj
index 5484060b63b..c93a4147416 100644
--- a/test/metabase/llm/provider_test.clj
+++ b/test/metabase/llm/provider_test.clj
@@ -1,5 +1,6 @@
(ns metabase.llm.provider-test
(:require
+ [clojure.string :as str]
[clojure.test :refer [deftest is testing use-fixtures]]
[metabase.llm.provider :as llm.provider]
[metabase.llm.settings :as llm.settings]
@@ -86,6 +87,48 @@
:secret-access-key "rotated-secret"
:region "us-west-1"})))))
+(deftest ^:parallel merge-config-preserves-a-masked-multi-line-secret-test
+ (testing (str "a service account key file is JSON that ends with a newline, so its mask straddles a line break — "
+ "echoing it back still has to keep the stored key rather than store the mask")
+ (let [key-file "{\n \"type\": \"service_account\",\n \"project_id\": \"my-project\"\n}\n"]
+ (is (= {:auth-method "service-account-key"
+ :service-account-key key-file}
+ (llm.provider/merge-config "google"
+ {:auth-method "service-account-key"
+ :service-account-key key-file}
+ {:auth-method "service-account-key"
+ :service-account-key (setting/obfuscate-value key-file)}))))))
+
+(deftest set-single-provider-setting!-ignores-a-masked-multi-line-secret-test
+ (testing (str "the setter trims before storing, but the mask of a newline-terminated secret only matches "
+ "untrimmed — echoing it back must keep the stored key rather than store the mask")
+ (let [key-file "{\n \"type\": \"service_account\",\n \"project_id\": \"my-project\"\n}\n"]
+ (mt/with-temporary-setting-values [llm-providers [(connection "google" "google"
+ {:auth-method "service-account-key"
+ :service-account-key key-file})]]
+ (llm.settings/llm-google-service-account-key! (setting/obfuscate-value key-file))
+ (is (= key-file (llm.settings/llm-google-service-account-key)))))))
+
+(deftest set-single-provider-setting!-ignores-a-whitespace-padded-mask-test
+ (testing "a mask that picked up surrounding whitespace in transit is still an echo, not a new value"
+ (let [key-file "{\"type\": \"service_account\", \"project_id\": \"my-project\"}"]
+ (mt/with-temporary-setting-values [llm-providers [(connection "google" "google"
+ {:auth-method "service-account-key"
+ :service-account-key key-file})]]
+ (llm.settings/llm-google-service-account-key! (str " " (setting/obfuscate-value key-file) " "))
+ (is (= key-file (llm.settings/llm-google-service-account-key)))))))
+
+(deftest set-single-provider-setting!-stores-a-fresh-value-test
+ (testing "a freshly entered value still replaces the stored one"
+ (let [old-key "{\"type\": \"service_account\", \"project_id\": \"old-project\"}"
+ new-key "{\"type\": \"service_account\", \"project_id\": \"new-project\"}\n"]
+ (mt/with-temporary-setting-values [llm-providers [(connection "google" "google"
+ {:auth-method "service-account-key"
+ :service-account-key old-key})]]
+ (llm.settings/llm-google-service-account-key! new-key)
+ (testing "trimmed, the way the setter has always stored"
+ (is (= (str/trim new-key) (llm.settings/llm-google-service-account-key))))))))
+
(deftest validate-config!-test
(testing "an unknown provider type is rejected"
(is (thrown-with-msg?
diff --git a/test/metabase/metabot/self/google/raw_predict_test.clj b/test/metabase/metabot/self/google/raw_predict_test.clj
new file mode 100644
index 00000000000..87155a58789
--- /dev/null
+++ b/test/metabase/metabot/self/google/raw_predict_test.clj
@@ -0,0 +1,158 @@
+(ns metabase.metabot.self.google.raw-predict-test
+ (:require
+ [clojure.test :refer :all]
+ [medley.core :as m]
+ [metabase.metabot.self.core :as self.core]
+ [metabase.metabot.self.google.raw-predict :as raw-predict]
+ [metabase.metabot.test-util :as metabot.tu]))
+
+(set! *warn-on-reflection* true)
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; request-body tests
+;;; ──────────────────────────────────────────────────────────────────
+
+(deftest ^:parallel request-body-drops-model-test
+ (testing "the streamRawPredict URL names the model, so the Messages body must not carry one"
+ (is (not (contains? (raw-predict/request-body "claude-haiku-4-5@20251001"
+ {:input [{:role :user :content "hi"}]})
+ :model)))))
+
+(deftest ^:parallel request-body-pins-anthropic-version-test
+ (testing "the platform takes `anthropic_version` in the body rather than the header the direct API uses"
+ (is (= @#'raw-predict/anthropic-version
+ (:anthropic_version (raw-predict/request-body "claude-haiku-4-5@20251001"
+ {:input [{:role :user :content "hi"}]}))))))
+
+(deftest ^:parallel request-body-is-otherwise-the-messages-body-test
+ (testing "everything but the model and the version is exactly what the direct Messages adapter builds"
+ (is (=? {:stream true
+ :cache_control {:type "ephemeral"}
+ :messages [{:role "user" :content [{:type "text" :text "hi"}]}]
+ :system [{:type "text" :text "You are terse." :cache_control {:type "ephemeral"}}]}
+ (raw-predict/request-body "claude-haiku-4-5@20251001"
+ {:system "You are terse."
+ :input [{:role :user :content "hi"}]})))))
+
+(deftest ^:parallel request-body-dated-model-resolves-max-tokens-test
+ (testing "a model dated in the platform's `@` spelling resolves to the same max_tokens ceiling as its
+ direct-API `-` spelling, rather than falling back to the unknown-model default"
+ (is (= 32000
+ (:max_tokens (raw-predict/request-body "claude-opus-4-1@20250805"
+ {:input [{:role :user :content "hi"}]}))))))
+
+(deftest ^:parallel request-body-bare-model-resolves-max-tokens-test
+ (testing "an undated model ID is already in the direct-API spelling and resolves as-is"
+ (is (= 128000
+ (:max_tokens (raw-predict/request-body "claude-sonnet-4-6"
+ {:input [{:role :user :content "hi"}]}))))))
+
+(deftest ^:parallel request-body-unknown-model-gets-default-max-tokens-test
+ (testing "a model outside the Claude adapter's knowledge falls back to the safe default ceiling"
+ (is (= 64000
+ (:max_tokens (raw-predict/request-body "claude-not-a-real-model"
+ {:input [{:role :user :content "hi"}]}))))))
+
+(deftest ^:parallel request-body-explicit-max-tokens-wins-test
+ (testing "an explicit :max-tokens overrides the model's ceiling"
+ (is (= 4096
+ (:max_tokens (raw-predict/request-body "claude-opus-4-1@20250805"
+ {:input [{:role :user :content "hi"}]
+ :max-tokens 4096}))))))
+
+(deftest ^:parallel request-body-dated-model-thinking-test
+ (testing "a dated current-generation model still gets its thinking config"
+ (is (=? {:thinking {:type "adaptive" :display "summarized"}}
+ (raw-predict/request-body "claude-sonnet-4-6@20250929"
+ {:input [{:role :user :content "hi"}]})))))
+
+(deftest ^:parallel request-body-reasoning-disabled-test
+ (testing ":reasoning? false suppresses thinking for a model that would otherwise stream it"
+ (is (not (contains? (raw-predict/request-body "claude-sonnet-4-6@20250929"
+ {:input [{:role :user :content "hi"}]
+ :reasoning? false})
+ :thinking)))))
+
+(deftest ^:parallel request-body-tools-get-cache-breakpoint-test
+ (testing "tools are converted and carry the prompt-caching breakpoint on the last one"
+ (is (=? {:tools [{:name "get-time"
+ :description "Return current time for a given IANA timezone."
+ :input_schema {:type "object"}
+ :cache_control {:type "ephemeral"}}]}
+ (raw-predict/request-body "claude-haiku-4-5@20251001"
+ {:input [{:role :user :content "hi"}]
+ :tools [(metabot.tu/get-time-tool)]})))))
+
+(deftest ^:parallel request-body-schema-forces-structured-output-test
+ (testing "a :schema becomes the forced structured_output tool, replacing any other tools"
+ (is (=? {:tool_choice {:type "tool" :name "structured_output"}
+ :tools [{:name "structured_output"
+ :input_schema {:type "object"}}]}
+ (raw-predict/request-body "claude-haiku-4-5@20251001"
+ {:input [{:role :user :content "hi"}]
+ :tools [(metabot.tu/get-time-tool)]
+ :schema {:type "object"}})))))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; ->aisdk-chunks-xf tests
+;;; ──────────────────────────────────────────────────────────────────
+
+(def ^:private text-stream-events
+ "A minimal Messages API text stream, which the platform serves verbatim."
+ [{:type "message_start" :message {:id "msg-1" :model "claude-haiku-4-5" :usage {:input_tokens 12 :output_tokens 0}}}
+ {:type "content_block_start" :index 0 :content_block {:type "text"}}
+ {:type "content_block_delta" :index 0 :delta {:type "text_delta" :text "Hello"}}
+ {:type "content_block_stop" :index 0}
+ {:type "message_delta" :delta {:stop_reason "end_turn"} :usage {:input_tokens 12 :output_tokens 3}}
+ {:type "message_stop"}])
+
+(deftest ^:parallel ->aisdk-chunks-xf-test
+ (testing "streamed Messages events are translated by the Claude adapter's transducer"
+ (is (=? [{:type :start} {:type :text-start} {:type :text-delta} {:type :text-end} {:type :usage}]
+ (into [] (comp (raw-predict/->aisdk-chunks-xf) (m/distinct-by :type)) text-stream-events)))))
+
+(deftest ^:parallel ->aisdk-chunks-xf-full-pipeline-test
+ (testing "through the full pipeline the stream produces the text part and its usage"
+ (is (=? [{:type :start :id string?}
+ {:type :text :text "Hello"}
+ {:type :usage :model "claude-haiku-4-5"
+ :usage {:promptTokens 12 :completionTokens 3}}]
+ (into [] (comp (raw-predict/->aisdk-chunks-xf) (self.core/aisdk-xf)) text-stream-events)))))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; reasoning-model? tests
+;;; ──────────────────────────────────────────────────────────────────
+
+(deftest ^:parallel reasoning-model?-test
+ (testing "reasoning support follows the Claude adapter's answer for the model"
+ (is (true? (raw-predict/reasoning-model? "claude-sonnet-4-6")))
+ (is (false? (raw-predict/reasoning-model? "claude-haiku-4-5")))))
+
+(deftest ^:parallel reasoning-model?-dated-model-test
+ (testing "a model dated in the platform's `@` spelling is recognized as the model it dates"
+ (is (true? (raw-predict/reasoning-model? "claude-sonnet-4-6@20250929")))
+ (is (false? (raw-predict/reasoning-model? "claude-haiku-4-5@20251001")))))
+
+(deftest ^:parallel reasoning-model?-unknown-model-test
+ (testing "a model we know nothing about does not stream reasoning"
+ (is (false? (raw-predict/reasoning-model? "claude-not-a-real-model")))
+ (is (false? (raw-predict/reasoning-model? nil)))))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; context-window-tokens tests
+;;; ──────────────────────────────────────────────────────────────────
+
+(deftest ^:parallel context-window-tokens-test
+ (testing "the context window follows the Claude adapter's answer for the model"
+ (is (= 1000000 (raw-predict/context-window-tokens "claude-sonnet-4-6")))
+ (is (= 200000 (raw-predict/context-window-tokens "claude-haiku-4-5-20251001")))))
+
+(deftest ^:parallel context-window-tokens-dated-model-test
+ (testing "a model dated in the platform's `@` spelling is recognized as the model it dates"
+ (is (= 200000 (raw-predict/context-window-tokens "claude-haiku-4-5@20251001")))
+ (is (= 200000 (raw-predict/context-window-tokens "claude-sonnet-4-5@20250929")))))
+
+(deftest ^:parallel context-window-tokens-unknown-model-test
+ (testing "a model we know nothing about has no context window"
+ (is (nil? (raw-predict/context-window-tokens "claude-not-a-real-model")))
+ (is (nil? (raw-predict/context-window-tokens nil)))))
diff --git a/test/metabase/metabot/self/google_test.clj b/test/metabase/metabot/self/google_test.clj
index 7cd4643c5f9..8fd58e4a772 100644
--- a/test/metabase/metabot/self/google_test.clj
+++ b/test/metabase/metabot/self/google_test.clj
@@ -4,17 +4,17 @@
[clojure.core.memoize :as memoize]
[clojure.string :as str]
[clojure.test :refer :all]
+ [metabase.llm.provider :as llm.provider]
[metabase.llm.settings :as llm.settings]
- [metabase.llm.test-util :as llm.tu]
[metabase.metabot.self.core :as self.core]
[metabase.metabot.self.debug :as debug]
[metabase.metabot.self.google :as google]
- [metabase.metabot.settings :as metabot.settings]
[metabase.test :as mt]
[metabase.util.json :as json])
(:import
(com.google.auth.oauth2 GoogleCredentials ServiceAccountCredentials)
(java.io IOException)
+ (java.net SocketTimeoutException)
(java.security KeyPairGenerator)
(java.util Base64)))
@@ -73,6 +73,35 @@
"google/gemini-3.7-flash" 1048576
"google/gemini-unknown" nil)))
+(deftest context-window-tokens-anthropic-test
+ (testing "an Anthropic partner model takes the window the Messages API adapter records for its model,
+ including when the platform dates it in the `@` spelling"
+ (are [model window] (= window (google/context-window-tokens model))
+ "anthropic/claude-fable-5" 1000000
+ "anthropic/claude-opus-5" 1000000
+ "anthropic/claude-opus-4-6" 1000000
+ "anthropic/claude-sonnet-5" 1000000
+ "anthropic/claude-sonnet-4-6" 1000000
+ "anthropic/claude-haiku-4-5@20251001" 200000
+ "anthropic/claude-unknown" nil)))
+
+(deftest context-window-tokens-unqualified-test
+ (testing "a model with no publisher qualifier is not treated as an Anthropic one"
+ (is (nil? (google/context-window-tokens "claude-sonnet-4-6")))
+ (is (nil? (google/context-window-tokens nil)))))
+
+(deftest context-window-tokens-unsupported-publisher-test
+ (testing "an unsupported publisher answers nil rather than throwing"
+ (is (nil? (google/context-window-tokens "mistralai/mistral-large")))
+ (is (nil? (google/context-window-tokens "Google/gemini-3.5-flash")))))
+
+(deftest every-offered-model-has-a-context-window-test
+ (testing "every model the provider registry offers has a context window under the id the registry uses"
+ (is (= [] (into []
+ (comp (map :id)
+ (remove google/context-window-tokens))
+ (llm.provider/fixed-models "google"))))))
+
;;; ──────────────────────────────────────────────────────────────────
;;; Auth / HTTP tests
;;; ──────────────────────────────────────────────────────────────────
@@ -83,9 +112,10 @@
llm.settings/llm-google-service-account-key nil
llm.settings/llm-google-project-id "my-project"
llm.settings/llm-google-location nil]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
(is (=? {:method :post
:url (str "https://aiplatform.googleapis.com/v1/projects/my-project/locations/global"
"/publishers/google/models/gemini-3.5-flash:streamGenerateContent?alt=sse")
@@ -99,9 +129,10 @@
llm.settings/llm-google-service-account-key nil
llm.settings/llm-google-project-id "my-project"
llm.settings/llm-google-location nil]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
(is (=? {:url (str "https://aiplatform.googleapis.com/v1/projects/my-project/locations/global"
"/publishers/google/models/gemini-3.5-flash:streamGenerateContent?alt=sse")}
(google-raw {:input [{:role :user :content "hi"}]})))))))
@@ -112,9 +143,10 @@
llm.settings/llm-google-service-account-key nil
llm.settings/llm-google-project-id "my-project"
llm.settings/llm-google-location nil]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
(let [req (google-raw {:model "google/gemini-3.5-flash"
:input [{:role :user :content "hi"}]})]
(is (=? {:as :stream
@@ -129,9 +161,10 @@
llm.settings/llm-google-service-account-key nil
llm.settings/llm-google-project-id "my-project"
llm.settings/llm-google-location "us-central1"]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
(is (=? {:url (str "https://us-central1-aiplatform.googleapis.com/v1/projects/my-project"
"/locations/us-central1/publishers/google/models"
"/gemini-3.5-flash:streamGenerateContent?alt=sse")}
@@ -144,9 +177,10 @@
llm.settings/llm-google-service-account-key nil
llm.settings/llm-google-project-id "my-project"
llm.settings/llm-google-location location]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
(is (=? {:url (format (str "https://aiplatform.%s.rep.googleapis.com"
"/v1/projects/my-project/locations/%s"
"/publishers/google/models/gemini-3.5-flash:streamGenerateContent?alt=sse")
@@ -241,26 +275,111 @@
llm.settings/llm-google-project-id "my-project"
llm.settings/llm-google-location "us-central1"
llm.settings/llm-google-api-base-url "https://gemini.proxy.example.com"]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
(is (=? {:url (str "https://gemini.proxy.example.com/v1/projects/my-project/locations/us-central1"
"/publishers/google/models/gemini-3.5-flash:streamGenerateContent?alt=sse")}
(google-raw {:model "google/gemini-3.5-flash" :input [{:role :user :content "hi"}]})))))))
-(deftest google-raw-non-google-publisher-model-test
- (testing "the publisher comes from the model ID's {publisher}/{model} qualifier"
+(deftest google-raw-unsupported-publisher-rejected-test
+ (testing "a Model Garden publisher this adapter cannot speak to is rejected rather than sent a Gemini body"
(mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
llm.settings/llm-google-service-account-key nil
llm.settings/llm-google-project-id "my-project"
llm.settings/llm-google-location nil]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})]
- (is (=? {:url (str "https://aiplatform.googleapis.com/v1/projects/my-project/locations/global"
- "/publishers/anthropic/models/claude-sonnet-4-6:streamGenerateContent?alt=sse")}
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]
+ (doseq [model ["mistralai/mistral-large"
+ "meta/llama-4-scout"
+ "qwen/qwen3-next"
+ "Google/gemini-3.5-flash"
+ "../../../v1beta1/evil"]]
+ (let [e (try (google-raw {:model model :input [{:role :user :content "hi"}]})
+ nil
+ (catch Exception e e))]
+ (is (= (str "Unsupported Google model " (pr-str model)
+ ". Only google/* and anthropic/* models are supported.")
+ (ex-message e)))
+ (is (=? {:api-error true
+ :status-code 400
+ :error-code :unsupported-model
+ :model model}
+ (ex-data e)))))))))
+
+(deftest list-models-unsupported-publisher-rejected-test
+ (testing "connecting rejects an unsupported publisher before any HTTP call"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Unsupported Google model \"mistralai/mistral-large\"\. Only google/\* and anthropic/\* models are supported\."
+ (list-models {:model "mistralai/mistral-large"})))))))
+
+(deftest google-raw-anthropic-model-stream-raw-predict-test
+ (testing "an anthropic model is served by its own streamRawPredict method rather than streamGenerateContent"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
+ (is (=? {:method :post
+ :url (str "https://aiplatform.googleapis.com/v1/projects/my-project/locations/global"
+ "/publishers/anthropic/models/claude-sonnet-4-6:streamRawPredict")
+ :headers {"Authorization" "Bearer ya29.pasted-access-token"}}
(google-raw {:model "anthropic/claude-sonnet-4-6" :input [{:role :user :content "hi"}]})))))))
+(deftest google-raw-anthropic-request-body-test
+ (testing "an anthropic model gets the Anthropic Messages body, with the model in the URL instead of the body and
+ the platform's pinned anthropic_version in its place"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
+ (let [req (google-raw {:model "anthropic/claude-haiku-4-5@20251001"
+ :system "You are terse."
+ :input [{:role :user :content "hi"}]})
+ body (json/decode+kw (:body req))]
+ (is (=? {:as :stream
+ :headers {"Content-Type" "application/json"}}
+ req))
+ (is (=? {:anthropic_version "vertex-2023-10-16"
+ :stream true
+ :messages [{:role "user" :content [{:type "text" :text "hi"}]}]
+ :system [{:type "text" :text "You are terse." :cache_control {:type "ephemeral"}}]}
+ body))
+ (is (not (contains? body :model))
+ "the URL names the model; a model in the body is rejected by the platform"))))))
+
+(deftest google-raw-anthropic-max-tokens-test
+ (testing "the Anthropic model whitelist supplies max_tokens for a bare current-generation ID, and the @-versioned
+ spelling of a dated ID resolves to the same entry the direct Anthropic adapter uses"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
+ (doseq [[model max-tokens] {"anthropic/claude-sonnet-4-6" 128000
+ "anthropic/claude-fable-5" 128000
+ "anthropic/claude-haiku-4-5@20251001" 64000}]
+ (testing model
+ (is (= max-tokens
+ (:max_tokens (json/decode+kw (:body (google-raw {:model model
+ :input [{:role :user :content "hi"}]}))))))))))))
+
(def ^:private bare-model-id-message-re
"The message [[metabase.metabot.self.google/model-resource-path]] throws for an unqualified model ID."
#"Invalid Google model \"gemini-3\.5-flash\" — expected a publisher-qualified ID like \"google/gemini-3\.5-flash\"")
@@ -297,12 +416,11 @@
llm.settings/llm-google-location nil]
(mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]
(doseq [model ["google/../../../v1beta1/evil"
- "../../../v1beta1/evil"
"google/models/gemini-3.5-flash"
"google/gemini-3.5-flash?alt=json"
"google/gemini-3.5-flash#"
"google/gemini 3.5 flash"
- "Google/gemini-3.5-flash"]]
+ "anthropic/claude sonnet 4-6"]]
(let [e (try (google-raw {:model model :input [{:role :user :content "hi"}]})
nil
(catch Exception e e))]
@@ -320,11 +438,12 @@
llm.settings/llm-google-service-account-key nil
llm.settings/llm-google-project-id "my-project"
llm.settings/llm-google-location nil]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})]
(is (=? {:url (str "https://aiplatform.googleapis.com/v1/projects/my-project/locations/global"
- "/publishers/anthropic/models/claude-sonnet-4-5@20250929:streamGenerateContent?alt=sse")}
+ "/publishers/anthropic/models/claude-sonnet-4-5@20250929:streamRawPredict")}
(google-raw {:model "anthropic/claude-sonnet-4-5@20250929"
:input [{:role :user :content "hi"}]})))))))
@@ -356,10 +475,11 @@
llm.settings/llm-google-service-account-key sa-key
llm.settings/llm-google-project-id nil
llm.settings/llm-google-location nil]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})
- google/fresh-bearer-headers (constantly {"Authorization" "Bearer test-sa-token"})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})
+ google/fresh-bearer-headers (constantly {"Authorization" "Bearer test-sa-token"})]
(is (=? {:url (str "https://aiplatform.googleapis.com/v1/projects/json-project/locations/global"
"/publishers/google/models/gemini-3.5-flash:streamGenerateContent?alt=sse")
:headers {"Authorization" "Bearer test-sa-token"}}
@@ -372,10 +492,11 @@
llm.settings/llm-google-service-account-key sa-key
llm.settings/llm-google-project-id "explicit-project"
llm.settings/llm-google-location nil]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})
- google/fresh-bearer-headers (constantly {"Authorization" "Bearer test-sa-token"})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})
+ google/fresh-bearer-headers (constantly {"Authorization" "Bearer test-sa-token"})]
(is (=? {:url (str "https://aiplatform.googleapis.com/v1/projects/explicit-project/locations/global"
"/publishers/google/models/gemini-3.5-flash:streamGenerateContent?alt=sse")}
(google-raw {:model "google/gemini-3.5-flash" :input [{:role :user :content "hi"}]}))))))))
@@ -387,10 +508,11 @@
llm.settings/llm-google-service-account-key sa-key
llm.settings/llm-google-project-id nil
llm.settings/llm-google-location nil]
- (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
- debug/capture-stream (fn [r _] r)
- http/request (fn [req] {:body req})
- google/fresh-bearer-headers (constantly {"Authorization" "Bearer test-sa-token"})]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] {:body req})
+ google/fresh-bearer-headers (constantly {"Authorization" "Bearer test-sa-token"})]
(is (=? {:headers {"Authorization" "Bearer test-sa-token"}}
(google-raw {:model "google/gemini-3.5-flash" :input [{:role :user :content "hi"}]}))))))))
@@ -523,17 +645,18 @@
(defn- aisdk-parts-for!
"The AISDK parts [[metabase.metabot.self.google/google]] yields for an SSE stream of `events`."
- [events]
- (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
- llm.settings/llm-google-service-account-key nil
- llm.settings/llm-google-project-id "my-project"
- llm.settings/llm-google-location nil]
- (mt/with-dynamic-fn-redefs [debug/capture-stream (fn [r _] r)
- http/request (fn [_] (sse-response-for events))]
- (into []
- (self.core/aisdk-xf)
- (google {:model "google/gemini-3.5-flash"
- :input [{:role :user :content "hi"}]})))))
+ ([events] (aisdk-parts-for! "google/gemini-3.5-flash" events))
+ ([model events]
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [debug/capture-stream (fn [r _] r)
+ http/request (fn [_] (sse-response-for events))]
+ (into []
+ (self.core/aisdk-xf)
+ (google {:model model
+ :input [{:role :user :content "hi"}]}))))))
(deftest google-text-stream-test
(testing "streamed text off the wire arrives as one coalesced text part with usage"
@@ -563,6 +686,79 @@
:finishReason "STOP"}]
:usageMetadata {:promptTokenCount 8 :candidatesTokenCount 4}}])))))
+(deftest google-anthropic-text-stream-test
+ (testing "an anthropic model's SSE events off the wire are translated by the Claude chunk translation"
+ (is (=? [{:type :start :id "msg_vrtx_011"}
+ {:type :text :text "Hello"}
+ {:type :usage
+ :model "claude-haiku-4-5-20251001"
+ :usage {:promptTokens 17 :completionTokens 5}}]
+ (aisdk-parts-for!
+ "anthropic/claude-haiku-4-5@20251001"
+ [{:type "message_start" :message {:id "msg_vrtx_011"
+ :model "claude-haiku-4-5-20251001"
+ :usage {:input_tokens 17 :output_tokens 1}}}
+ {:type "content_block_start" :index 0 :content_block {:type "text" :text ""}}
+ {:type "content_block_delta" :index 0 :delta {:type "text_delta" :text "Hel"}}
+ {:type "content_block_delta" :index 0 :delta {:type "text_delta" :text "lo"}}
+ {:type "content_block_stop" :index 0}
+ {:type "message_delta"
+ :delta {:stop_reason "end_turn"}
+ :usage {:input_tokens 17 :output_tokens 5}}
+ {:type "message_stop"}])))))
+
+(deftest google-anthropic-tool-call-stream-test
+ (testing "an anthropic model's streamed tool_use block arrives as a tool-input part with parsed arguments"
+ (is (=? [{:type :start :id "msg_vrtx_012"}
+ {:type :tool-input
+ :function "get_time"
+ :arguments {:tz "UTC"}}
+ {:type :usage :usage {:promptTokens 8 :completionTokens 4}}]
+ (aisdk-parts-for!
+ "anthropic/claude-haiku-4-5@20251001"
+ [{:type "message_start" :message {:id "msg_vrtx_012"
+ :model "claude-haiku-4-5-20251001"
+ :usage {:input_tokens 8 :output_tokens 1}}}
+ {:type "content_block_start" :index 0 :content_block {:type "tool_use"
+ :id "toolu_01"
+ :name "get_time"}}
+ {:type "content_block_delta" :index 0 :delta {:type "input_json_delta" :partial_json "{\"tz\":"}}
+ {:type "content_block_delta" :index 0 :delta {:type "input_json_delta" :partial_json "\"UTC\"}"}}
+ {:type "content_block_stop" :index 0}
+ {:type "message_delta"
+ :delta {:stop_reason "tool_use"}
+ :usage {:input_tokens 8 :output_tokens 4}}
+ {:type "message_stop"}])))))
+
+;;; ──────────────────────────────────────────────────────────────────
+;;; reasoning-model? tests
+;;; ──────────────────────────────────────────────────────────────────
+
+(deftest reasoning-model?-anthropic-test
+ (testing "an Anthropic partner model reasons when the Messages API adapter says its model does"
+ (is (true? (google/reasoning-model? "anthropic/claude-sonnet-4-6")))
+ (is (false? (google/reasoning-model? "anthropic/claude-haiku-4-5")))))
+
+(deftest reasoning-model?-anthropic-platform-spelling-test
+ (testing "a dated partner model is recognized in the platform's `@` spelling"
+ (is (true? (google/reasoning-model? "anthropic/claude-sonnet-4-6@20250929")))
+ (is (false? (google/reasoning-model? "anthropic/claude-haiku-4-5@20251001")))))
+
+(deftest reasoning-model?-gemini-test
+ (testing "Gemini sends thinking only as :thought parts, which the adapter drops"
+ (is (false? (google/reasoning-model? "google/gemini-3.5-flash")))
+ (is (false? (google/reasoning-model? "google/gemini-3.6-pro")))))
+
+(deftest reasoning-model?-unqualified-test
+ (testing "a model with no publisher qualifier is not treated as an Anthropic one"
+ (is (false? (google/reasoning-model? "claude-sonnet-4-6")))
+ (is (false? (google/reasoning-model? nil)))))
+
+(deftest reasoning-model?-unsupported-publisher-test
+ (testing "an unsupported publisher answers false rather than throwing"
+ (is (false? (google/reasoning-model? "mistralai/mistral-large")))
+ (is (false? (google/reasoning-model? "Google/gemini-3.5-flash")))))
+
;;; ──────────────────────────────────────────────────────────────────
;;; list-models tests
;;; ──────────────────────────────────────────────────────────────────
@@ -593,6 +789,132 @@
:body (json/encode {:contents [{:role "user" :parts [{:text "hi"}]}]})}]
@calls)))))))
+(def ^:private anthropic-validation-error-body
+ "Anthropic error for empty probe body when the request reached the model."
+ (json/encode {:type "error"
+ :error {:type "invalid_request_error" :message "messages: Field required"}}))
+
+(def ^:private not-servable-in-region-body
+ "Google's error envelope for a model the location does not serve."
+ (json/encode {:error {:code 400
+ :message (str "Publisher Model `projects/my-project/locations/us-central1/publishers/anthropic"
+ "/models/claude-haiku-4-5@20251001` is not servable in region us-central1.")
+ :status "FAILED_PRECONDITION"}}))
+
+(defn- stub-error
+ "An `http/request` stub that throws an exception with `status` and `body`, recording every request into `calls`."
+ [calls status body]
+ (fn [req]
+ (swap! calls conj req)
+ (throw (ex-info (str "clj-http: status " status)
+ {:status status
+ :headers {"content-type" "application/json"}
+ :body body}))))
+
+(deftest list-models-anthropic-empty-body-probe-test
+ (testing "an anthropic model is probed by posting an empty body to its own streamRawPredict route"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (let [calls (atom [])]
+ (mt/with-dynamic-fn-redefs [http/request (stub-error calls 400 anthropic-validation-error-body)]
+ (is (= {:models []}
+ (list-models {:model "anthropic/claude-haiku-4-5@20251001"})))
+ (is (=? [{:method :post
+ :url (str "https://aiplatform.googleapis.com/v1/projects/my-project/locations/global"
+ "/publishers/anthropic/models/claude-haiku-4-5@20251001:streamRawPredict")
+ :headers {"Authorization" "Bearer ya29.pasted-access-token"}
+ :body "{}"}]
+ @calls)))))))
+
+(deftest list-models-anthropic-spends-no-tokens-test
+ (testing "the anthropic probe body names no messages, so it cannot reach inference"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (let [calls (atom [])]
+ (mt/with-dynamic-fn-redefs [http/request (stub-error calls 400 anthropic-validation-error-body)]
+ (list-models {:model "anthropic/claude-haiku-4-5@20251001"})
+ (is (= {} (json/decode+kw (:body (first @calls))))))))))
+
+(deftest list-models-anthropic-not-servable-in-region-rejected-test
+ (testing "a 400 in Google's error format means the location does not serve the model, and is surfaced"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location "us-central1"]
+ (mt/with-dynamic-fn-redefs [http/request (stub-error (atom []) 400 not-servable-in-region-body)]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Google API rejected the request as invalid"
+ (list-models {:model "anthropic/claude-haiku-4-5@20251001"})))))))
+
+(deftest list-models-anthropic-model-not-found-rejected-test
+ (testing "a 404 for a model this project cannot reach is surfaced rather than swallowed"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [http/request (stub-error (atom []) 404 (json/encode {:error {:code 404 :message "Publisher model was not found"}}))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Google API endpoint is unavailable or the model was not found"
+ (list-models {:model "anthropic/claude-sonnet-4-6"})))))))
+
+(deftest list-models-anthropic-unauthenticated-rejected-test
+ (testing "a 401 from an expired credential is surfaced rather than read as a model verdict"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.expired"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [http/request (stub-error (atom []) 401 (json/encode {:error {:code 401 :message "invalid authentication credentials"}}))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Google API credentials expired or invalid"
+ (list-models {:model "anthropic/claude-haiku-4-5@20251001"})))))))
+
+(deftest list-models-anthropic-unexpected-success-accepted-test
+ (testing "a 2xx also proves the model resolved, even though the empty body should never earn one"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body "{}"})]
+ (is (= {:models []}
+ (list-models {:model "anthropic/claude-haiku-4-5@20251001"})))))))
+
+(deftest list-models-anthropic-probe-uses-inference-method-test
+ (testing "the probe and inference hit the same streamRawPredict verb, so the probe validates what will run"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (let [probe-calls (atom [])
+ inference-calls (atom [])]
+ (mt/with-dynamic-fn-redefs [http/request (stub-error probe-calls 400 anthropic-validation-error-body)]
+ (list-models {:model "anthropic/claude-haiku-4-5@20251001"}))
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible identity
+ self.core/reducible-with-api-errors (fn [r _ _] r)
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [req] (swap! inference-calls conj req) {:body req})]
+ (google-raw {:model "anthropic/claude-haiku-4-5@20251001" :input [{:role :user :content "hi"}]}))
+ (is (= (:url (first @inference-calls))
+ (:url (first @probe-calls))))))))
+
+(deftest list-models-anthropic-invalid-model-rejected-test
+ (testing "an anthropic model whose ID cannot be a path segment is rejected before any HTTP call"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Invalid Google model \"anthropic/claude bad model\""
+ (list-models {:model "anthropic/claude bad model"})))))))
+
(deftest list-models-bare-model-throws-test
(testing "a bare model ID throws before any HTTP call"
(mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
@@ -605,30 +927,48 @@
bare-model-id-message-re
(list-models {:model "gemini-3.5-flash"})))))))
-(deftest list-models-defaults-to-saved-model-test
- (testing "without a model in opts the probe runs against the model the saved reference names"
- (llm.tu/with-connections [(llm.tu/connection "google")]
- (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
- llm.settings/llm-google-service-account-key nil
- llm.settings/llm-google-project-id "my-project"
- llm.settings/llm-google-location nil
- metabot.settings/llm-metabot-provider "google/google/gemini-3.6-flash"]
- (let [calls (atom [])]
- (mt/with-dynamic-fn-redefs [http/request (stub-count-tokens calls)]
- (is (= {:models []} (list-models)))
- (is (=? [{:url (str "https://aiplatform.googleapis.com/v1/projects/my-project/locations/global"
- "/publishers/google/models/gemini-3.6-flash:countTokens")}]
- @calls))))))))
-
(deftest list-models-no-model-skips-probe-test
- (testing "with no candidate model and a non-Google provider configured no call is made"
+ (testing "with no candidate model there is nothing to probe and no call is made"
(mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
llm.settings/llm-google-service-account-key nil
- llm.settings/llm-google-project-id "my-project"
- metabot.settings/llm-metabot-provider "anthropic/claude-sonnet-4-6"]
+ llm.settings/llm-google-project-id "my-project"]
(mt/with-dynamic-fn-redefs [http/request (fn [_] (throw (ex-info "should never be called" {})))]
(is (= {:models []} (list-models)))))))
+(deftest list-models-reports-the-probed-model-test
+ (testing "the model the probe verified is reported back, for the connection to be re-verified against later"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [http/request (stub-count-tokens (atom []))]
+ (is (= {:models []
+ :learned-config {:probed-model "google/gemini-3.5-flash"}}
+ (list-models {:model "google/gemini-3.5-flash" :probe? true})))))))
+
+(deftest list-models-anthropic-reports-the-probed-model-test
+ (testing "an Anthropic partner model is reported in the spelling the platform names it by, `@` date and all"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [http/request (stub-error (atom []) 400 anthropic-validation-error-body)]
+ (is (= {:models []
+ :learned-config {:probed-model "anthropic/claude-haiku-4-5@20251001"}}
+ (list-models {:model "anthropic/claude-haiku-4-5@20251001" :probe? true})))))))
+
+(deftest list-models-without-probe-keeps-the-model-to-itself-test
+ (testing "a plain listing still validates the credentials but reports no `:learned-config` to the client"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.pasted-access-token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (let [requests (atom [])]
+ (mt/with-dynamic-fn-redefs [http/request (stub-count-tokens requests)]
+ (is (= {:models []}
+ (list-models {:model "google/gemini-3.5-flash"})))
+ (is (= 1 (count @requests))))))))
+
(deftest list-models-explicit-credentials-test
(testing "credentials in opts override the configured settings"
(mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.saved-token"
@@ -785,6 +1125,48 @@
"check that \"nowhere1\" is a valid location")
(ex-message e))))))))
+(deftest google-raw-mid-stream-failure-is-translated-test
+ (testing "a failure while consuming the stream surfaces as a tagged Google error, not a raw IOException"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible (fn [_]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ _rf _init]
+ (throw (IOException. "Connection reset")))))
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [_] {:body nil})]
+ (let [e (try (into [] (google-raw {:model "google/gemini-3.5-flash"
+ :input [{:role :user :content "hi"}]}))
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (=? {:api-error true
+ :provider "google"
+ :error-code :provider-request-failed}
+ (ex-data e)))
+ (is (= "google API request failed: Connection reset" (ex-message e))))))))
+
+(deftest google-raw-anthropic-mid-stream-failure-is-translated-test
+ (testing "an Anthropic partner model's stream gets the same translation as a Gemini model's"
+ (mt/with-temporary-setting-values [llm.settings/llm-google-oauth-access-token "ya29.token"
+ llm.settings/llm-google-service-account-key nil
+ llm.settings/llm-google-project-id "my-project"
+ llm.settings/llm-google-location nil]
+ (mt/with-dynamic-fn-redefs [self.core/sse-reducible (fn [_]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ _rf _init]
+ (throw (SocketTimeoutException. "Read timed out")))))
+ debug/capture-stream (fn [r _] r)
+ http/request (fn [_] {:body nil})]
+ (let [e (try (into [] (google-raw {:model "anthropic/claude-haiku-4-5@20251001"
+ :input [{:role :user :content "hi"}]}))
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (=? {:api-error true
+ :provider "google"
+ :error-code :provider-request-failed}
+ (ex-data e)))
+ (is (= "google API request failed: Read timed out" (ex-message e))))))))
+
(deftest list-models-invalid-request-maps-to-google-error-test
(testing "a 400 from the countTokens probe surfaces the canonical message with the upstream detail"
(let [upstream-message "Request contains an invalid argument."]
diff --git a/test/metabase/metabot/self/vllm_test.clj b/test/metabase/metabot/self/vllm_test.clj
index 43b99d27f8b..f4bce46b8bc 100644
--- a/test/metabase/metabot/self/vllm_test.clj
+++ b/test/metabase/metabot/self/vllm_test.clj
@@ -756,8 +756,8 @@
(deftest preflight-passes-on-a-correctly-configured-server-test
(testing "a server that returns a well-formed tool call passes and still lists its models"
(is (= {:models [{:id "vllm-test" :display_name "vllm-test"}]
- :probed-model "vllm-test"
- :learned-config {vllm/reasoning-config-key "false"}}
+ :learned-config {vllm/reasoning-config-key "false"
+ :probed-model "vllm-test"}}
(probe! [{:id "vllm-test" :max_model_len 32768}] tool-calling-message)))))
(deftest preflight-reports-the-model-it-probed-test
@@ -765,10 +765,12 @@
the connect path must adopt the model the contract checks actually ran against"
(testing "the first catalog entry, in the order the server lists it"
(is (= "vllm-test"
- (:probed-model (probe! recorded-multi-model-catalog tool-calling-message)))))
+ (get-in (probe! recorded-multi-model-catalog tool-calling-message)
+ [:learned-config :probed-model]))))
(testing "including a slash-bearing repo id, which the connect path stores as `vllm/{id}`"
(is (= "mlx-community/Qwen3-14B-4bit"
- (:probed-model (probe! recorded-no-alias-catalog tool-calling-message))))))
+ (get-in (probe! recorded-no-alias-catalog tool-calling-message)
+ [:learned-config :probed-model])))))
(testing "and a listing that did not probe reports nothing"
(mt/with-dynamic-fn-redefs [http/request (fn [_] {:status 200 :body {:data [{:id "vllm-test"}]}})]
(is (= [:models] (keys (vllm/list-models {:credentials credentials})))))))
@@ -788,7 +790,7 @@
:max_model_len 32768}]}}
(do (reset! probed (:model (json/decode+kw (str body))))
{:status 200 :body {:choices [{:message tool-calling-message}]}})))]
- (is (= "vllm-test" (:probed-model (vllm/list-models {:credentials credentials :probe? true}))))
+ (is (= "vllm-test" (get-in (vllm/list-models {:credentials credentials :probe? true}) [:learned-config :probed-model])))
(is (= "vllm-test" @probed)))))
(testing "an explicitly requested adapter is still honoured — the filter only picks the default"
(mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url]}]
@@ -797,9 +799,10 @@
:body {:data [{:id "sql-lora" :parent "vllm-test" :max_model_len 32768}
{:id "vllm-test" :max_model_len 32768}]}}
{:status 200 :body {:choices [{:message tool-calling-message}]}}))]
- (is (= "sql-lora" (:probed-model (vllm/list-models {:credentials credentials
- :probe? true
- :model "sql-lora"})))))))
+ (is (= "sql-lora" (get-in (vllm/list-models {:credentials credentials
+ :probe? true
+ :model "sql-lora"})
+ [:learned-config :probed-model]))))))
(deftest preflight-skipped-without-probe-flag-test
(testing "without :probe? no generation request is made at all — listing models must stay cheap"
@@ -863,6 +866,39 @@
(vllm/list-models {:credentials credentials :probe? true :model "second"})
(is (= "second" @probed))))))
+(deftest preflight-prefers-a-proposed-model-that-is-still-served-test
+ (testing "an edit without an explicit model re-probes the model previously verified for the connection"
+ (let [probed (atom #{})]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200 :body {:data [{:id "first" :max_model_len 32768}
+ {:id "previous" :max_model_len 32768}]}}
+ (do (swap! probed conj (:model (json/decode+kw (str body))))
+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]
+ (is (= "previous"
+ (get-in (vllm/list-models {:credentials credentials
+ :probe? true
+ :proposed-model "previous"})
+ [:learned-config :probed-model])))
+ (is (= #{"previous"} @probed))))))
+
+(deftest preflight-falls-back-when-the-proposed-model-is-no-longer-served-test
+ (testing "a stale proposal does not become a hard request and the normal candidate selection still applies"
+ (let [probed (atom #{})]
+ (mt/with-dynamic-fn-redefs [http/request (fn [{:keys [url body]}]
+ (if (re-find #"/models$" (str url))
+ {:status 200
+ :body {:data [{:id "sql-lora" :parent "first" :max_model_len 32768}
+ {:id "first" :max_model_len 32768}]}}
+ (do (swap! probed conj (:model (json/decode+kw (str body))))
+ {:status 200 :body {:choices [{:message tool-calling-message}]}})))]
+ (is (= "first"
+ (get-in (vllm/list-models {:credentials credentials
+ :probe? true
+ :proposed-model "removed"})
+ [:learned-config :probed-model])))
+ (is (= #{"first"} @probed))))))
+
(deftest preflight-rejects-a-model-the-server-does-not-serve-test
(testing "a requested model absent from the catalog fails and names what is served, rather than probing something else"
(let [generated? (atom false)]
@@ -958,22 +994,22 @@
(testing "the probe is the only place this is knowable — /v1/models carries no reasoning field, so what it
saw is reported back for the connection to record"
;; Recorded shape: vLLM 0.26 puts it on the non-streaming message under `reasoning`.
- (is (= {vllm/reasoning-config-key "true"}
- (:learned-config
- (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
- {:message (assoc tool-calling-message
- :reasoning "\nOkay, the user wants me to record \"orders\".")
- :finish_reason "tool_calls"})))))
+ (is (=? {vllm/reasoning-config-key "true"}
+ (:learned-config
+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
+ {:message (assoc tool-calling-message
+ :reasoning "\nOkay, the user wants me to record \"orders\".")
+ :finish_reason "tool_calls"})))))
(testing "and under the deprecated spelling older servers still use"
- (is (= {vllm/reasoning-config-key "true"}
- (:learned-config
- (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
- {:message (assoc tool-calling-message
- :reasoning_content "Older vLLM builds spell it this way.")
- :finish_reason "tool_calls"})))))
+ (is (=? {vllm/reasoning-config-key "true"}
+ (:learned-config
+ (probe-choice! [{:id "vllm-test" :max_model_len 32768}]
+ {:message (assoc tool-calling-message
+ :reasoning_content "Older vLLM builds spell it this way.")
+ :finish_reason "tool_calls"})))))
(testing "a model that answers without reasoning reports false"
- (is (= {vllm/reasoning-config-key "false"}
- (:learned-config (probe! [{:id "vllm-test" :max_model_len 32768}] tool-calling-message)))))
+ (is (=? {vllm/reasoning-config-key "false"}
+ (:learned-config (probe! [{:id "vllm-test" :max_model_len 32768}] tool-calling-message)))))
(testing "and a connection carrying the flag is what the request body reads it from"
(is (true? (vllm/reasoning-connection? {vllm/reasoning-config-key "true"})))
(is (false? (vllm/reasoning-connection? {vllm/reasoning-config-key "false"})))
@@ -1086,8 +1122,8 @@
(testing "`max_model_len` is vLLM's own field: Ollama, LM Studio, and TGI omit it. The floor is
best-effort, so a catalog without it connects rather than being rejected on a missing field"
(is (= {:models [{:id "vllm-test" :display_name "vllm-test"}]
- :probed-model "vllm-test"
- :learned-config {vllm/reasoning-config-key "false"}}
+ :learned-config {vllm/reasoning-config-key "false"
+ :probed-model "vllm-test"}}
(probe! [{:id "vllm-test"}] tool-calling-message)))))
(deftest preflight-does-not-leak-the-context-window-to-the-client-test
diff --git a/test/metabase/metabot/settings_test.clj b/test/metabase/metabot/settings_test.clj
index b53e152c817..e04416b19cc 100644
--- a/test/metabase/metabot/settings_test.clj
+++ b/test/metabase/metabot/settings_test.clj
@@ -18,6 +18,9 @@
(def ^:private configured-anthropic
(connection "anthropic" "anthropic" {:api-key "sk-ant-test"}))
+(def ^:private configured-google
+ (connection "google" "google" {:oauth-access-token "ya29.test" :project-id "my-project"}))
+
(defn- do-with-connections!
[conns thunk]
(mt/with-temporary-setting-values [llm-providers conns]
@@ -184,13 +187,18 @@
(testing "only anthropic and openai models that stream reasoning report support"
(with-connections [(connection "anthropic" "anthropic")
(connection "openai" "openai")
- (connection "bedrock" "bedrock")]
+ (connection "bedrock" "bedrock")
+ (connection "google" "google")]
(doseq [[model-ref expected]
- {"anthropic/claude-sonnet-4-6" true
- "anthropic/claude-haiku-4-5" false
- "openai/gpt-5.4" true
- "openai/gpt-4o" false
- "bedrock/anthropic.claude-opus-4-8" false}]
+ {"anthropic/claude-sonnet-4-6" true
+ "anthropic/claude-haiku-4-5" false
+ "openai/gpt-5.4" true
+ "openai/gpt-4o" false
+ "bedrock/anthropic.claude-opus-4-8" false
+ ;; google serves both wire families; only its Claude models stream reasoning back
+ "google/anthropic/claude-sonnet-4-6" true
+ "google/anthropic/claude-haiku-4-5@20251001" false
+ "google/google/gemini-3.5-flash" false}]
(testing model-ref
(with-selected-model model-ref
(is (= expected (metabot.settings/llm-metabot-supports-reasoning?)))))))))
@@ -312,6 +320,35 @@
clojure.lang.ExceptionInfo #"Invalid Azure model"
(metabot.settings/llm-metabot-provider! "azure/anthropic/a/b"))))))
+(deftest validate-metabot-provider-google-model-format-test
+ (with-connections [configured-anthropic configured-google]
+ (testing "accepts a publisher-qualified model"
+ (mt/with-temporary-setting-values [llm-metabot-provider "google/google/gemini-3.5-flash"]
+ (is (= "google/google/gemini-3.5-flash" (metabot.settings/llm-metabot-provider))))
+ (mt/with-temporary-setting-values [llm-metabot-provider "google/anthropic/claude-haiku-4-5@20251001"]
+ (is (= "google/anthropic/claude-haiku-4-5@20251001" (metabot.settings/llm-metabot-provider)))))))
+
+(deftest validate-metabot-provider-google-rejects-an-unqualified-model-test
+ (with-connections [configured-anthropic configured-google]
+ (testing "rejects a model with no publisher: the connection key is not one, so this names the model \"gemini-3.5-flash\""
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"Invalid Google model \"google/gemini-3.5-flash\""
+ (metabot.settings/llm-metabot-provider! "google/gemini-3.5-flash"))))))
+
+(deftest validate-metabot-provider-google-rejects-an-unsupported-publisher-test
+ (with-connections [configured-anthropic configured-google]
+ (testing "rejects a publisher this provider does not serve"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"Invalid Google model \"google/evilai/some-model\""
+ (metabot.settings/llm-metabot-provider! "google/evilai/some-model"))))))
+
+(deftest validate-metabot-provider-google-rejects-a-slash-in-the-model-id-test
+ (with-connections [configured-anthropic configured-google]
+ (testing "rejects a model ID with a slash in it, which is not one path segment"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"Invalid Google model \"google/anthropic/a/b\""
+ (metabot.settings/llm-metabot-provider! "google/anthropic/a/b"))))))
+
(deftest validate-metabot-provider-managed-model-allow-list-test
(mt/with-premium-features #{:metabase-ai-managed}
(with-connections [configured-anthropic (connection "metabase" "metabase")]
diff --git a/test/metabase/settings/models/setting_test.clj b/test/metabase/settings/models/setting_test.clj
index 94945cb9fad..f9fe68b1238 100644
--- a/test/metabase/settings/models/setting_test.clj
+++ b/test/metabase/settings/models/setting_test.clj
@@ -734,6 +734,27 @@
(is (= "123456"
(test-sensitive-setting)))))
+(defsetting test-sensitive-multi-line-setting
+ (deferred-tru "This is a sample sensitive multi-line Setting.")
+ :sensitive? true)
+
+(deftest sensitive-settings-multi-line-test
+ (testing "`user-facing-value` should obfuscate multi-line sensitive settings"
+ (test-sensitive-multi-line-setting! "ABC1\n3")
+ (is (= "**********\n3"
+ (setting/user-facing-value "test-sensitive-multi-line-setting"))))
+ (testing "Attempting to set a sensitive setting to a multi-line obfuscated value should be ignored"
+ (test-sensitive-multi-line-setting! "1234\n6")
+ (test-sensitive-multi-line-setting! "**********\n6")
+ (is (= "1234\n6"
+ (test-sensitive-multi-line-setting)))))
+
+(deftest obfuscated-value-spanning-a-line-break-test
+ (testing "obfuscated-value? works for values ending in a newline"
+ (let [key-file "{\n \"type\": \"service_account\"\n}\n"]
+ (is (= "**********}\n" (setting/obfuscate-value key-file)))
+ (is (setting/obfuscated-value? (setting/obfuscate-value key-file))))))
+
;;; ------------------------------------------------- CACHE SYNCING --------------------------------------------------
(deftest cache-sync-test