Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Record which provider served each LLM call (#82280)

Andrei Simionescud37f1a9dd33f
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj b/enterprise/backend/src/metabase_enterprise/metabot/usage.clj
index 4304476828b..34e55dfd264 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot/usage.clj
@@ -54,8 +54,8 @@
(defenterprise log-ai-usage!
"Record an LLM API call in the ai_usage_log table."
:feature :none
- [{:keys [source model prompt-tokens completion-tokens cache-creation-tokens cache-read-tokens
- user-id tenant-id conversation-id profile-id request-id ai-proxied]}]
+ [{:keys [source model provider model-name prompt-tokens completion-tokens cache-creation-tokens
+ cache-read-tokens user-id tenant-id conversation-id profile-id request-id ai-proxied]}]
(when-not (contains? known-sources source)
(throw (ex-info (str "Unknown ai_usage_log source " (pr-str source))
{:source source})))
@@ -71,6 +71,8 @@
(metabot.db/insert-usage-log!
{:source source
:model model
+ :provider provider
+ :model_name model-name
:prompt_tokens prompt-tokens
:completion_tokens completion-tokens
:total_tokens total-tokens
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj
index 61823fed41e..deae30fdecd 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj
@@ -43,7 +43,9 @@
(let [before-count (t2/count :model/AiUsageLog :user_id user-id :source "metabot_agent")]
(usage/log-ai-usage!
{:source "metabot_agent"
- :model "anthropic/claude-test"
+ :model "openrouter-1/anthropic/claude-test"
+ :provider "openrouter"
+ :model-name "anthropic/claude-test"
:prompt-tokens 100
:completion-tokens 50
:ai-proxied true})
@@ -52,7 +54,9 @@
(t2/count :model/AiUsageLog :user_id user-id :source "metabot_agent")))
(let [row (t2/select-one :model/AiUsageLog :user_id user-id :source "metabot_agent"
{:order-by [[:id :desc]]})]
- (is (= "anthropic/claude-test" (:model row)))
+ (is (= "openrouter-1/anthropic/claude-test" (:model row)))
+ (is (= "openrouter" (:provider row)))
+ (is (= "anthropic/claude-test" (:model_name row)))
(is (= 100 (:prompt_tokens row)))
(is (= 50 (:completion_tokens row)))
(is (= 150 (:total_tokens row)))
diff --git a/resources/migrations/065/20260910_ai_usage_log_provider.yaml b/resources/migrations/065/20260910_ai_usage_log_provider.yaml
new file mode 100644
index 00000000000..146df23d51d
--- /dev/null
+++ b/resources/migrations/065/20260910_ai_usage_log_provider.yaml
@@ -0,0 +1,40 @@
+# Record which provider served each call, and the model as that provider names it
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.2026-09-10T12:00:00
+ author: andreis
+ comment: Add provider to ai_usage_log
+ changes:
+ - addColumn:
+ tableName: ai_usage_log
+ columns:
+ - column:
+ name: provider
+ remarks: 'Provider type that served the call, e.g. openrouter. A managed-proxy call records the wire family it was served as, so it reads anthropic and is told apart by ai_proxied'
+ type: varchar(100)
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: ai_usage_log
+ columnName: provider
+
+ - changeSet:
+ id: v65.2026-09-10T12:00:01
+ author: andreis
+ comment: Add model_name to ai_usage_log
+ changes:
+ - addColumn:
+ tableName: ai_usage_log
+ columns:
+ - column:
+ name: model_name
+ remarks: 'Model as the provider names it, without the connection key, e.g. anthropic/claude-haiku-4.5'
+ type: varchar(255)
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: ai_usage_log
+ columnName: model_name
diff --git a/src/metabase/analytics/llm_token_usage.clj b/src/metabase/analytics/llm_token_usage.clj
index 471c0b803db..69483f00d34 100644
--- a/src/metabase/analytics/llm_token_usage.clj
+++ b/src/metabase/analytics/llm_token_usage.clj
@@ -60,11 +60,12 @@
user-id))
(def ^:private prometheus-arg-keys
- [:model-id :tag :prompt-tokens :completion-tokens :cache-creation-tokens :cache-read-tokens])
+ [:model-id :provider :tag :prompt-tokens :completion-tokens :cache-creation-tokens :cache-read-tokens])
(def ^:private PrometheusArgs
[:map {:closed true}
[:model-id :string]
+ [:provider {:optional true} [:maybe :string]]
[:tag :string]
[:prompt-tokens ms/IntGreaterThanOrEqualToZero]
[:completion-tokens ms/IntGreaterThanOrEqualToZero]
@@ -73,10 +74,10 @@
(mu/defn track-prometheus!
"Track Prometheus LLM token usage metrics."
- [{:keys [model-id tag prompt-tokens completion-tokens
+ [{:keys [model-id provider tag prompt-tokens completion-tokens
cache-creation-tokens cache-read-tokens]}
:- PrometheusArgs]
- (let [labels {:model model-id :source tag}]
+ (let [labels {:model model-id :source tag :provider (or provider "unknown")}]
(analytics/inc! :metabase-metabot/llm-input-tokens labels prompt-tokens)
(analytics/inc! :metabase-metabot/llm-output-tokens labels completion-tokens)
(when (and cache-creation-tokens (pos? cache-creation-tokens))
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index eab04887eb6..9932b16f88a 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -744,35 +744,38 @@
{:description (str "Number of Slack bot responses that reached the user as nothing at all, "
"because the plain-text fallback failed too.")})
;; metabot / LLM agent metrics
+ ;; The `:provider` label is one of the provider types in `metabase.llm.provider`, or `unknown` if a caller omits it.
+ ;; Resolution rejects anything else, so the set only grows when a type joins the registry.
+ ;; It is a projection of `:model`, which already carries the connection key, so it adds no series.
(prometheus/counter :metabase-metabot/llm-requests
{:description "LLM provider API requests"
- :labels [:model :source]})
+ :labels [:model :source :provider]})
(prometheus/counter :metabase-metabot/llm-retries
{:description "LLM provider retry attempts"
- :labels [:model :source]})
+ :labels [:model :source :provider]})
(prometheus/counter :metabase-metabot/llm-errors
{:description "LLM provider API errors (excluding retries)"
- :labels [:model :source :error-type]})
+ :labels [:model :source :provider :error-type]})
(prometheus/histogram :metabase-metabot/llm-duration-ms
{:description "LLM request duration (ms)"
- :labels [:model :source]
+ :labels [:model :source :provider]
;; 100 ms -> 2 minutes
:buckets [100 500 1000 2000 5000 10000 20000 30000 60000 120000]})
(prometheus/counter :metabase-metabot/llm-input-tokens
{:description "LLM input tokens"
- :labels [:model :source]})
+ :labels [:model :source :provider]})
(prometheus/counter :metabase-metabot/llm-output-tokens
{:description "LLM output tokens"
- :labels [:model :source]})
+ :labels [:model :source :provider]})
(prometheus/counter :metabase-metabot/llm-cache-creation-tokens
{:description "LLM cache creation input tokens (Anthropic prompt caching)"
- :labels [:model :source]})
+ :labels [:model :source :provider]})
(prometheus/counter :metabase-metabot/llm-cache-read-tokens
{:description "LLM cache read input tokens (Anthropic prompt caching)"
- :labels [:model :source]})
+ :labels [:model :source :provider]})
(prometheus/histogram :metabase-metabot/llm-tokens-per-call
{:description "Tokens per LLM call"
- :labels [:model :source]
+ :labels [:model :source :provider]
:buckets [1000 2500 5000 10000 20000 50000 100000 200000]})
(prometheus/counter :metabase-metabot/agent-requests
{:description "Agent loop invocations"
diff --git a/src/metabase/llm/api.clj b/src/metabase/llm/api.clj
index 892d5e8fb7c..8d9137a4f8c 100644
--- a/src/metabase/llm/api.clj
+++ b/src/metabase/llm/api.clj
@@ -224,6 +224,7 @@
:user-id api/*current-user-id*
:request-id (analytics/uuid->ai-service-hex-uuid (random-uuid))
:model-id (:model usage)
+ :provider "anthropic"
:prompt-tokens (:prompt usage)
:completion-tokens (:completion usage)
:total-tokens (+ (:prompt usage) (:completion usage))
@@ -235,6 +236,8 @@
(metabot/log-ai-usage!
{:source "sql-gen"
:model (:model usage)
+ :provider "anthropic"
+ :model-name (:model usage)
:prompt-tokens (:prompt usage)
:completion-tokens (:completion usage)})
(track-sqlgen-event!
diff --git a/src/metabase/metabot/schema.clj b/src/metabase/metabot/schema.clj
index 7bbbc5348ec..656403b29ad 100644
--- a/src/metabase/metabot/schema.clj
+++ b/src/metabase/metabot/schema.clj
@@ -230,6 +230,8 @@
[:created_at {:optional true} [:maybe ms/TemporalInstant]]
[:source {:optional true} [:maybe [:or :keyword :string]]]
[:model {:optional true} [:maybe [:or :keyword :string]]]
+ [:provider {:optional true} [:maybe :string]]
+ [:model_name {:optional true} [:maybe :string]]
[:prompt_tokens {:optional true} [:maybe :int]]
[:completion_tokens {:optional true} [:maybe :int]]
[:total_tokens {:optional true} [:maybe :int]]
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 4ab5c2b2e1b..f49ace975d1 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -261,6 +261,11 @@
backoff-ms)
jitter)))
+(defn- provider-label
+ "The `:provider` label on an LLM call's metrics: the type of the connection serving it, so `metabase` when proxied."
+ [{:keys [provider ai-proxy?]}]
+ (if ai-proxy? "metabase" provider))
+
(defn- report-aisdk-errors-xf
"Transducer that logs and increments the llm-errors counter for :error parts in the aisdk stream."
[tracking-opts]
@@ -276,23 +281,33 @@
(analytics/inc! :metabase-metabot/llm-errors
{:model (:model tracking-opts "unknown")
:source (:tag tracking-opts "none")
+ :provider (provider-label tracking-opts)
:error-type "llm-sse-error"}))
part)))
(defn- report-token-usage-xf
"Transducer that reports token_usage metrics for :usage parts in the aisdk stream.
+ Every field goes to [[metabase.metabot.usage/log-ai-usage!]], where `:tag` stands in for a missing `:source`.
+
Prometheus + Snowplow:
- - `:profile-id` — the profile id (e.g. `:internal`)
- - `:model` — the model (e.g. `openrouter/anthropic/claude-haiku-4.5`)
- - `:tag` — the specific purpose for which the tokens were used (e.g. 'agent', 'sql-fixing')
-
- Snowplow only:
- - `:request-id` — UUID string for this request
- - `:session-id` — conversation UUID string
- - `:source` — the source of the request (e.g., 'metabot_agent', 'document_generate_content').
- Indicates which API endpoint or workflow initiated the LLM call."
- [{:keys [model profile-id request-id session-id source tag ai-proxy?]}]
+ - `:model` - the model reference (e.g. `openrouter/anthropic/claude-haiku-4.5`)
+ - `:tag` - the specific purpose for which the tokens were used (e.g. 'agent', 'sql-fixing')
+
+ Prometheus only:
+ - `:provider` - the provider type serving it (e.g. `openrouter`)
+ - `:ai-proxy?` - whether the call went through the managed AI proxy
+
+ Snowplow only:
+ - `:profile-id` - the profile id (e.g. `:internal`)
+ - `:request-id` - UUID string for this request
+ - `:session-id` - conversation UUID string
+ - `:source` - the source of the request (e.g., 'metabot_agent', 'document_generate_content').
+ Indicates which API endpoint or workflow initiated the LLM call.
+
+ Neither:
+ - `:model-name` - the model as the provider names it (e.g. `anthropic/claude-haiku-4.5`)"
+ [{:keys [model model-name provider profile-id request-id session-id source tag ai-proxy?] :as tracking-opts}]
(let [start-ms (u/start-timer)]
(map (fn [part]
(when (= (:type part) :usage)
@@ -308,6 +323,7 @@
:snowplow (some? request-id)
:profile (some-> profile-id name)
:model-id model
+ :provider (provider-label tracking-opts)
:prompt-tokens prompt
:completion-tokens completion
:cache-creation-tokens cache-creation
@@ -323,6 +339,8 @@
(usage/log-ai-usage!
{:source (or source tag "unknown")
:model model
+ :provider provider
+ :model-name model-name
:prompt-tokens prompt
:completion-tokens completion
:cache-creation-tokens cache-creation
@@ -358,7 +376,7 @@
(defn- with-retries
"Execute `(thunk)` with retry logic for transient LLM errors.
Retries up to `max-llm-retries` attempts with exponential backoff.
- Records prometheus metrics with `:model` and `:tag` from `tracking-opts` as labels.
+ Records prometheus metrics with `:model` and `:tag` from `tracking-opts`, and its [[provider-label]], as labels.
`retry?` is an optional predicate on the caught exception, ANDed with
[[retryable-error?]]; returning false surfaces the error without retrying. The
@@ -366,7 +384,9 @@
([tracking-opts thunk]
(with-retries tracking-opts thunk (constantly true)))
([tracking-opts thunk retry?]
- (let [labels {:model (:model tracking-opts) :source (:tag tracking-opts)}]
+ (let [labels {:model (:model tracking-opts)
+ :source (:tag tracking-opts)
+ :provider (provider-label tracking-opts)}]
(loop [attempt 1]
(analytics/inc! :metabase-metabot/llm-requests labels)
(let [timer (u/start-timer)
@@ -504,7 +524,8 @@
(let [{:keys [provider stream-fn model credentials ai-proxy?]} (parse-provider-model provider-and-model)]
(log/info "Calling LLM" {:provider provider :model model :parts (count parts) :tools (count tools)
:tool-choice tool-choice :ai-proxy? ai-proxy?})
- (let [tracking-opts (assoc tracking-opts :model provider-and-model :ai-proxy? ai-proxy?)
+ (let [tracking-opts (assoc tracking-opts :model provider-and-model :provider provider
+ :model-name model :ai-proxy? ai-proxy?)
streaming-opts (cond-> {:model model :input parts :tools (vals tools)
:credentials credentials :ai-proxy? ai-proxy?
:fast? (metabot.settings/llm-fast-mode)}
@@ -576,7 +597,8 @@
:ai-proxy? ai-proxy?})
tracking-opts (-> opts
(dissoc :required-permission)
- (assoc :model provider-and-model :ai-proxy? ai-proxy?))
+ (assoc :model provider-and-model :provider provider :model-name model
+ :ai-proxy? ai-proxy?))
streaming-opts (cond-> {:model model
:input input
:schema json-schema
diff --git a/src/metabase/metabot/usage.clj b/src/metabase/metabot/usage.clj
index 81de97c2be0..6bcc7e4815f 100644
--- a/src/metabase/metabot/usage.clj
+++ b/src/metabase/metabot/usage.clj
@@ -19,6 +19,8 @@
[:map
[:source :string]
[:model :string]
+ [:provider {:optional true} [:maybe :string]]
+ [:model-name {:optional true} [:maybe :string]]
[:prompt-tokens [:int {:min 0}]]
[:completion-tokens [:int {:min 0}]]
[:cache-creation-tokens {:optional true} [:maybe [:int {:min 0}]]]
diff --git a/test/metabase/analytics/llm_token_usage_test.clj b/test/metabase/analytics/llm_token_usage_test.clj
index 3be227154be..eb3be216705 100644
--- a/test/metabase/analytics/llm_token_usage_test.clj
+++ b/test/metabase/analytics/llm_token_usage_test.clj
@@ -93,9 +93,10 @@
(deftest track-prometheus!-test
(mt/with-prometheus-system! [_ system]
- (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag"}]
+ (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag" :provider "anthropic"}]
(testing "increments prometheus metrics with correct labels and values"
(llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"
+ :provider "anthropic"
:tag "test-tag"
:prompt-tokens 100
:completion-tokens 50})
@@ -108,6 +109,7 @@
(clear-llm-metrics!)
(testing "positive cache token fields increment their counters"
(llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"
+ :provider "anthropic"
:tag "test-tag"
:prompt-tokens 100
:completion-tokens 50
@@ -116,8 +118,27 @@
(is (= 400.0 (mt/metric-value system :metabase-metabot/llm-cache-creation-tokens labels)))
(is (= 1600.0 (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels))))
(clear-llm-metrics!)
+ (testing "a call with no provider is counted under the unknown provider"
+ (llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"
+ :tag "test-tag"
+ :prompt-tokens 100
+ :completion-tokens 50})
+ (is (= 100.0 (mt/metric-value system :metabase-metabot/llm-input-tokens
+ (assoc labels :provider "unknown")))))
+ (clear-llm-metrics!)
+ (testing "the provider label separates series that would otherwise share a model and source"
+ (llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"
+ :provider "openrouter"
+ :tag "test-tag"
+ :prompt-tokens 100
+ :completion-tokens 50})
+ (is (zero? (mt/metric-value system :metabase-metabot/llm-input-tokens labels)))
+ (is (= 100.0 (mt/metric-value system :metabase-metabot/llm-input-tokens
+ (assoc labels :provider "openrouter")))))
+ (clear-llm-metrics!)
(testing "zero / nil cache token fields do not increment their counters"
(llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"
+ :provider "anthropic"
:tag "test-tag"
:prompt-tokens 100
:completion-tokens 50
@@ -139,6 +160,7 @@
:prometheus true
:request-id "req-123"
:model-id "anthropic/claude-haiku-4-5"
+ :provider "anthropic"
:tag "test-tag"
:prompt-tokens 100
:completion-tokens 50
@@ -151,7 +173,7 @@
"prompt_tokens" 100}}]
(snowplow-test/pop-event-data-and-user-id!))))
(testing "Prometheus metrics incremented"
- (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag"}]
+ (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag" :provider "anthropic"}]
(is (= 100.0 (mt/metric-value system :metabase-metabot/llm-input-tokens labels)))
(is (= 50.0 (mt/metric-value system :metabase-metabot/llm-output-tokens labels)))
(is (= 150.0 (:sum (mt/metric-value system :metabase-metabot/llm-tokens-per-call labels)))))))))
@@ -171,7 +193,7 @@
(is (empty? (snowplow-test/pop-event-data-and-user-id!))))
(testing "Prometheus still fires"
(is (= 100.0 (mt/metric-value system :metabase-metabot/llm-input-tokens
- {:model "anthropic/claude-haiku-4-5" :source "test-tag"}))))))
+ {:model "anthropic/claude-haiku-4-5" :source "test-tag" :provider "unknown"}))))))
(clear-llm-metrics!)
(testing "Prometheus suppressed when :prometheus false"
(mt/with-temporary-setting-values [premium-embedding-token nil
@@ -190,7 +212,7 @@
(snowplow-test/pop-event-data-and-user-id!))))
(testing "no Prometheus metrics incremented"
(is (= 0.0 (mt/metric-value system :metabase-metabot/llm-input-tokens
- {:model "openai/gpt-4" :source "none"})))))))))
+ {:model "openai/gpt-4" :source "none" :provider "unknown"})))))))))
(deftest track-token-usage!-both-false-error-test
(testing "throws when both :snowplow and :prometheus are false"
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index 54b65f4d0e8..88b5277565d 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -914,7 +914,8 @@
{:profile-id "internal"})))
(is (== 2 (mt/metric-value system :metabase-metabot/llm-requests
{:model "openrouter/anthropic/claude-haiku-4-5"
- :source "agent"})))
+ :source "agent"
+ :provider "openrouter"})))
(is (== 1 (:count (mt/metric-value system :metabase-metabot/agent-duration-ms
{:profile-id "internal"}))))
(is (pos? (:sum (mt/metric-value system :metabase-metabot/agent-duration-ms
@@ -940,7 +941,8 @@
{:profile-id "internal"})))
(is (== 1 (mt/metric-value system :metabase-metabot/llm-requests
{:model "openrouter/anthropic/claude-haiku-4-5"
- :source "agent"})))
+ :source "agent"
+ :provider "openrouter"})))
(is (== 1 (:count (mt/metric-value system :metabase-metabot/agent-duration-ms
{:profile-id "internal"}))))
(is (pos? (:sum (mt/metric-value system :metabase-metabot/agent-duration-ms
diff --git a/test/metabase/metabot/api/document_test.clj b/test/metabase/metabot/api/document_test.clj
index 72dfc4d0ef9..ba929f2b992 100644
--- a/test/metabase/metabot/api/document_test.clj
+++ b/test/metabase/metabot/api/document_test.clj
@@ -72,11 +72,11 @@
(is (== 1 (:sum (mt/metric-value system :metabase-metabot/agent-iterations
{:profile-id "document-generate-content"}))))
(is (== 1 (mt/metric-value system :metabase-metabot/llm-requests
- {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))
+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent" :provider "openrouter"})))
(is (== 100 (mt/metric-value system :metabase-metabot/llm-input-tokens
- {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))
+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent" :provider "openrouter"})))
(is (== 20 (mt/metric-value system :metabase-metabot/llm-output-tokens
- {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"}))))))
+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent" :provider "openrouter"}))))))
(deftest generate-content-tool-call-produces-draft-card-test
(testing "a document_construct_sql_chart tool call round trip produces a :draft_card (#73690)"
diff --git a/test/metabase/metabot/example_question_generator_test.clj b/test/metabase/metabot/example_question_generator_test.clj
index e8f5086b8a7..1bc73e17520 100644
--- a/test/metabase/metabot/example_question_generator_test.clj
+++ b/test/metabase/metabot/example_question_generator_test.clj
@@ -151,7 +151,7 @@
(mt/with-prometheus-system! [_ system]
(mt/with-temporary-setting-values [llm-providers llm.tu/default-connections
llm-metabot-provider "openrouter/test-model"]
- (let [labels {:model "openrouter/test-model" :source "example-question-generation"}]
+ (let [labels {:model "openrouter/test-model" :source "example-question-generation" :provider "openrouter"}]
(testing "increments llm-requests and observes duration on success"
(mt/with-dynamic-fn-redefs [openrouter/openrouter
(constantly (test-util/mock-llm-response
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 9c58b0a8943..ab7791e20c3 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -77,6 +77,13 @@
(#'self/parse-provider-model "deepseek/deepseek-v4-flash")))
(is (=? {:provider "google" :model "google/gemini-3.5-flash" :ai-proxy? false}
(#'self/parse-provider-model "google/google/gemini-3.5-flash"))))
+ (testing "resolves the provider type, not the admin's name for the connection"
+ (llm.tu/with-connections [{:key "openrouter-1"
+ :type "openrouter"
+ :name "openrouter-1"
+ :config {:api-key "sk-or-v1-test"}}]
+ (is (=? {:provider "openrouter" :model "anthropic/claude-sonnet-4.6" :ai-proxy? false}
+ (#'self/parse-provider-model "openrouter-1/anthropic/claude-sonnet-4.6")))))
(testing "a vLLM served model is often a Hugging Face repo id, so the model segment keeps its slashes"
(llm.tu/with-connections [(llm.tu/connection "vllm")]
(is (=? {:provider "vllm" :model "mlx-community/Qwen3-14B-4bit" :ai-proxy? false}
@@ -1374,7 +1381,7 @@
(llm.tu/with-default-connections
(mt/with-prometheus-system! [_ system]
(mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)]
- (let [labels {:model "openrouter/test-model" :source "metabot_agent"}]
+ (let [labels {:model "openrouter/test-model" :source "metabot_agent" :provider "openrouter"}]
(testing "increments llm-requests and observes duration on success"
(mt/with-dynamic-fn-redefs [openrouter/openrouter (constantly (test-util/mock-llm-response [{:type :start :id "m1"}]))]
(run! identity (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"})))
@@ -1470,13 +1477,22 @@
:model "test-model"}]))]
(run! identity (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"})))
(is (zero? (mt/metric-value system :metabase-metabot/llm-cache-creation-tokens labels)))
- (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels)))))))))
+ (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels))))
+ (testing "labels a managed-proxy call with the metabase provider"
+ (mt/with-dynamic-fn-redefs [self.claude/claude
+ (constantly (test-util/mock-llm-response
+ [{:type :start :id "m1"}
+ {:type :usage :usage {:promptTokens 10 :completionTokens 5}}]))]
+ (run! identity (self/call-llm "metabase/anthropic/claude-haiku-4-5" nil [] {} {:tag "metabot_agent"})))
+ (let [managed-labels (assoc labels :model "metabase/anthropic/claude-haiku-4-5" :provider "metabase")]
+ (is (== 1 (mt/metric-value system :metabase-metabot/llm-requests managed-labels)))
+ (is (== 10 (mt/metric-value system :metabase-metabot/llm-input-tokens managed-labels))))))))))
(deftest call-llm-structured-prometheus-test
(llm.tu/with-default-connections
(mt/with-prometheus-system! [_ system]
(mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)]
- (let [labels {:model "openrouter/test-model" :source "metabot_agent"}
+ (let [labels {:model "openrouter/test-model" :source "metabot_agent" :provider "openrouter"}
success-mock (test-util/mock-llm-response
[{:type :start :id "m1"}
{:type :tool-input :id "call-1" :function "json"
@@ -1632,6 +1648,35 @@
"session_id" "00000000-0000-0000-0000-000000000002"}}]
token-events))))))))))
+;;; ===================== Usage Log Tests =====================
+
+(deftest call-llm-usage-log-test
+ (testing "call-llm and call-llm-structured log the provider type and the model as the provider names it"
+ (llm.tu/with-connections [(assoc (llm.tu/connection "openrouter") :key "openrouter-1")
+ (llm.tu/connection "metabase")]
+ (let [model-ref "openrouter-1/anthropic/claude-sonnet-4.6"
+ response (test-util/mock-llm-response
+ [{:type :start :id "m1"}
+ {:type :tool-input :id "call-1" :function "json" :arguments {:answer "42"}}
+ {:type :usage :usage {:promptTokens 10 :completionTokens 5}}])
+ logged (atom [])]
+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (constantly response)
+ self.claude/claude (constantly response)
+ usage/log-ai-usage! #(swap! logged conj %)]
+ (run! identity (self/call-llm model-ref nil [] {} {:tag "metabot_agent"}))
+ (self/call-llm-structured model-ref [{:role "user" :content "test"}]
+ {:type "object" :properties {:answer {:type "string"}}} 0.3 1024
+ {:tag "metabot_agent"})
+ (run! identity (self/call-llm "metabase/anthropic/claude-haiku-4-5" nil [] {} {:tag "metabot_agent"})))
+ (is (=? (concat (repeat 2 {:model model-ref
+ :provider "openrouter"
+ :model-name "anthropic/claude-sonnet-4.6"})
+ [{:model "metabase/anthropic/claude-haiku-4-5"
+ :provider "anthropic"
+ :model-name "claude-haiku-4-5"
+ :ai-proxied true}])
+ @logged))))))
+
;;; ----- gating: usage-limit + permission checks in call-llm-structured-with-trace -----
;;; (UXW-4126) The structured-with-trace path enforces usage limits unconditionally and
;;; an optional `:required-permission` against the current user's metabot perms.