The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Record which provider served each LLM call (#82280)
Andrei Simionescud37f1a9dd33f
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj b/enterprise/backend/src/metabase_enterprise/metabot/usage.cljindex 4304476828b..34e55dfd264 100644--- a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot/usage.clj@@ -54,8 +54,8 @@(defenterprise log-ai-usage!"Record an LLM API call in the ai_usage_log table.":feature :none- [{:keys [source model prompt-tokens completion-tokens cache-creation-tokens cache-read-tokens- user-id tenant-id conversation-id profile-id request-id ai-proxied]}]+ [{:keys [source model provider model-name prompt-tokens completion-tokens cache-creation-tokens+ cache-read-tokens user-id tenant-id conversation-id profile-id request-id ai-proxied]}](when-not (contains? known-sources source)(throw (ex-info (str "Unknown ai_usage_log source " (pr-str source)){:source source})))@@ -71,6 +71,8 @@(metabot.db/insert-usage-log!{:source source:model model+ :provider provider+ :model_name model-name:prompt_tokens prompt-tokens:completion_tokens completion-tokens:total_tokens total-tokensdiff --git a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.cljindex 61823fed41e..deae30fdecd 100644--- a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj@@ -43,7 +43,9 @@(let [before-count (t2/count :model/AiUsageLog :user_id user-id :source "metabot_agent")](usage/log-ai-usage!{:source "metabot_agent"- :model "anthropic/claude-test"+ :model "openrouter-1/anthropic/claude-test"+ :provider "openrouter"+ :model-name "anthropic/claude-test":prompt-tokens 100:completion-tokens 50:ai-proxied true})@@ -52,7 +54,9 @@(t2/count :model/AiUsageLog :user_id user-id :source "metabot_agent")))(let [row (t2/select-one :model/AiUsageLog :user_id user-id :source "metabot_agent"{:order-by [[:id :desc]]})]- (is (= "anthropic/claude-test" (:model row)))+ (is (= "openrouter-1/anthropic/claude-test" (:model row)))+ (is (= "openrouter" (:provider row)))+ (is (= "anthropic/claude-test" (:model_name row)))(is (= 100 (:prompt_tokens row)))(is (= 50 (:completion_tokens row)))(is (= 150 (:total_tokens row)))diff --git a/resources/migrations/065/20260910_ai_usage_log_provider.yaml b/resources/migrations/065/20260910_ai_usage_log_provider.yamlnew file mode 100644index 00000000000..146df23d51d--- /dev/null+++ b/resources/migrations/065/20260910_ai_usage_log_provider.yaml@@ -0,0 +1,40 @@+# Record which provider served each call, and the model as that provider names it++databaseChangeLog:+ - changeSet:+ id: v65.2026-09-10T12:00:00+ author: andreis+ comment: Add provider to ai_usage_log+ changes:+ - addColumn:+ tableName: ai_usage_log+ columns:+ - column:+ name: provider+ remarks: 'Provider type that served the call, e.g. openrouter. A managed-proxy call records the wire family it was served as, so it reads anthropic and is told apart by ai_proxied'+ type: varchar(100)+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: ai_usage_log+ columnName: provider++ - changeSet:+ id: v65.2026-09-10T12:00:01+ author: andreis+ comment: Add model_name to ai_usage_log+ changes:+ - addColumn:+ tableName: ai_usage_log+ columns:+ - column:+ name: model_name+ remarks: 'Model as the provider names it, without the connection key, e.g. anthropic/claude-haiku-4.5'+ type: varchar(255)+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: ai_usage_log+ columnName: model_namediff --git a/src/metabase/analytics/llm_token_usage.clj b/src/metabase/analytics/llm_token_usage.cljindex 471c0b803db..69483f00d34 100644--- a/src/metabase/analytics/llm_token_usage.clj+++ b/src/metabase/analytics/llm_token_usage.clj@@ -60,11 +60,12 @@user-id))(def ^:private prometheus-arg-keys- [:model-id :tag :prompt-tokens :completion-tokens :cache-creation-tokens :cache-read-tokens])+ [:model-id :provider :tag :prompt-tokens :completion-tokens :cache-creation-tokens :cache-read-tokens])(def ^:private PrometheusArgs[:map {:closed true}[:model-id :string]+ [:provider {:optional true} [:maybe :string]][:tag :string][:prompt-tokens ms/IntGreaterThanOrEqualToZero][:completion-tokens ms/IntGreaterThanOrEqualToZero]@@ -73,10 +74,10 @@(mu/defn track-prometheus!"Track Prometheus LLM token usage metrics."- [{:keys [model-id tag prompt-tokens completion-tokens+ [{:keys [model-id provider tag prompt-tokens completion-tokenscache-creation-tokens cache-read-tokens]}:- PrometheusArgs]- (let [labels {:model model-id :source tag}]+ (let [labels {:model model-id :source tag :provider (or provider "unknown")}](analytics/inc! :metabase-metabot/llm-input-tokens labels prompt-tokens)(analytics/inc! :metabase-metabot/llm-output-tokens labels completion-tokens)(when (and cache-creation-tokens (pos? cache-creation-tokens))diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex eab04887eb6..9932b16f88a 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -744,35 +744,38 @@{:description (str "Number of Slack bot responses that reached the user as nothing at all, ""because the plain-text fallback failed too.")});; metabot / LLM agent metrics+ ;; The `:provider` label is one of the provider types in `metabase.llm.provider`, or `unknown` if a caller omits it.+ ;; Resolution rejects anything else, so the set only grows when a type joins the registry.+ ;; It is a projection of `:model`, which already carries the connection key, so it adds no series.(prometheus/counter :metabase-metabot/llm-requests{:description "LLM provider API requests"- :labels [:model :source]})+ :labels [:model :source :provider]})(prometheus/counter :metabase-metabot/llm-retries{:description "LLM provider retry attempts"- :labels [:model :source]})+ :labels [:model :source :provider]})(prometheus/counter :metabase-metabot/llm-errors{:description "LLM provider API errors (excluding retries)"- :labels [:model :source :error-type]})+ :labels [:model :source :provider :error-type]})(prometheus/histogram :metabase-metabot/llm-duration-ms{:description "LLM request duration (ms)"- :labels [:model :source]+ :labels [:model :source :provider];; 100 ms -> 2 minutes:buckets [100 500 1000 2000 5000 10000 20000 30000 60000 120000]})(prometheus/counter :metabase-metabot/llm-input-tokens{:description "LLM input tokens"- :labels [:model :source]})+ :labels [:model :source :provider]})(prometheus/counter :metabase-metabot/llm-output-tokens{:description "LLM output tokens"- :labels [:model :source]})+ :labels [:model :source :provider]})(prometheus/counter :metabase-metabot/llm-cache-creation-tokens{:description "LLM cache creation input tokens (Anthropic prompt caching)"- :labels [:model :source]})+ :labels [:model :source :provider]})(prometheus/counter :metabase-metabot/llm-cache-read-tokens{:description "LLM cache read input tokens (Anthropic prompt caching)"- :labels [:model :source]})+ :labels [:model :source :provider]})(prometheus/histogram :metabase-metabot/llm-tokens-per-call{:description "Tokens per LLM call"- :labels [:model :source]+ :labels [:model :source :provider]:buckets [1000 2500 5000 10000 20000 50000 100000 200000]})(prometheus/counter :metabase-metabot/agent-requests{:description "Agent loop invocations"diff --git a/src/metabase/llm/api.clj b/src/metabase/llm/api.cljindex 892d5e8fb7c..8d9137a4f8c 100644--- a/src/metabase/llm/api.clj+++ b/src/metabase/llm/api.clj@@ -224,6 +224,7 @@:user-id api/*current-user-id*:request-id (analytics/uuid->ai-service-hex-uuid (random-uuid)):model-id (:model usage)+ :provider "anthropic":prompt-tokens (:prompt usage):completion-tokens (:completion usage):total-tokens (+ (:prompt usage) (:completion usage))@@ -235,6 +236,8 @@(metabot/log-ai-usage!{:source "sql-gen":model (:model usage)+ :provider "anthropic"+ :model-name (:model usage):prompt-tokens (:prompt usage):completion-tokens (:completion usage)})(track-sqlgen-event!diff --git a/src/metabase/metabot/schema.clj b/src/metabase/metabot/schema.cljindex 7bbbc5348ec..656403b29ad 100644--- a/src/metabase/metabot/schema.clj+++ b/src/metabase/metabot/schema.clj@@ -230,6 +230,8 @@[:created_at {:optional true} [:maybe ms/TemporalInstant]][:source {:optional true} [:maybe [:or :keyword :string]]][:model {:optional true} [:maybe [:or :keyword :string]]]+ [:provider {:optional true} [:maybe :string]]+ [:model_name {:optional true} [:maybe :string]][:prompt_tokens {:optional true} [:maybe :int]][:completion_tokens {:optional true} [:maybe :int]][:total_tokens {:optional true} [:maybe :int]]diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 4ab5c2b2e1b..f49ace975d1 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -261,6 +261,11 @@backoff-ms)jitter)))+(defn- provider-label+ "The `:provider` label on an LLM call's metrics: the type of the connection serving it, so `metabase` when proxied."+ [{:keys [provider ai-proxy?]}]+ (if ai-proxy? "metabase" provider))+(defn- report-aisdk-errors-xf"Transducer that logs and increments the llm-errors counter for :error parts in the aisdk stream."[tracking-opts]@@ -276,23 +281,33 @@(analytics/inc! :metabase-metabot/llm-errors{:model (:model tracking-opts "unknown"):source (:tag tracking-opts "none")+ :provider (provider-label tracking-opts):error-type "llm-sse-error"}))part)))(defn- report-token-usage-xf"Transducer that reports token_usage metrics for :usage parts in the aisdk stream.+ Every field goes to [[metabase.metabot.usage/log-ai-usage!]], where `:tag` stands in for a missing `:source`.+Prometheus + Snowplow:- - `:profile-id` — the profile id (e.g. `:internal`)- - `:model` — the model (e.g. `openrouter/anthropic/claude-haiku-4.5`)- - `:tag` — the specific purpose for which the tokens were used (e.g. 'agent', 'sql-fixing')-- Snowplow only:- - `:request-id` — UUID string for this request- - `:session-id` — conversation UUID string- - `:source` — the source of the request (e.g., 'metabot_agent', 'document_generate_content').- Indicates which API endpoint or workflow initiated the LLM call."- [{:keys [model profile-id request-id session-id source tag ai-proxy?]}]+ - `:model` - the model reference (e.g. `openrouter/anthropic/claude-haiku-4.5`)+ - `:tag` - the specific purpose for which the tokens were used (e.g. 'agent', 'sql-fixing')++ Prometheus only:+ - `:provider` - the provider type serving it (e.g. `openrouter`)+ - `:ai-proxy?` - whether the call went through the managed AI proxy++ Snowplow only:+ - `:profile-id` - the profile id (e.g. `:internal`)+ - `:request-id` - UUID string for this request+ - `:session-id` - conversation UUID string+ - `:source` - the source of the request (e.g., 'metabot_agent', 'document_generate_content').+ Indicates which API endpoint or workflow initiated the LLM call.++ Neither:+ - `:model-name` - the model as the provider names it (e.g. `anthropic/claude-haiku-4.5`)"+ [{:keys [model model-name provider profile-id request-id session-id source tag ai-proxy?] :as tracking-opts}](let [start-ms (u/start-timer)](map (fn [part](when (= (:type part) :usage)@@ -308,6 +323,7 @@:snowplow (some? request-id):profile (some-> profile-id name):model-id model+ :provider (provider-label tracking-opts):prompt-tokens prompt:completion-tokens completion:cache-creation-tokens cache-creation@@ -323,6 +339,8 @@(usage/log-ai-usage!{:source (or source tag "unknown"):model model+ :provider provider+ :model-name model-name:prompt-tokens prompt:completion-tokens completion:cache-creation-tokens cache-creation@@ -358,7 +376,7 @@(defn- with-retries"Execute `(thunk)` with retry logic for transient LLM errors.Retries up to `max-llm-retries` attempts with exponential backoff.- Records prometheus metrics with `:model` and `:tag` from `tracking-opts` as labels.+ Records prometheus metrics with `:model` and `:tag` from `tracking-opts`, and its [[provider-label]], as labels.`retry?` is an optional predicate on the caught exception, ANDed with[[retryable-error?]]; returning false surfaces the error without retrying. The@@ -366,7 +384,9 @@([tracking-opts thunk](with-retries tracking-opts thunk (constantly true)))([tracking-opts thunk retry?]- (let [labels {:model (:model tracking-opts) :source (:tag tracking-opts)}]+ (let [labels {:model (:model tracking-opts)+ :source (:tag tracking-opts)+ :provider (provider-label tracking-opts)}](loop [attempt 1](analytics/inc! :metabase-metabot/llm-requests labels)(let [timer (u/start-timer)@@ -504,7 +524,8 @@(let [{:keys [provider stream-fn model credentials ai-proxy?]} (parse-provider-model provider-and-model)](log/info "Calling LLM" {:provider provider :model model :parts (count parts) :tools (count tools):tool-choice tool-choice :ai-proxy? ai-proxy?})- (let [tracking-opts (assoc tracking-opts :model provider-and-model :ai-proxy? ai-proxy?)+ (let [tracking-opts (assoc tracking-opts :model provider-and-model :provider provider+ :model-name model :ai-proxy? ai-proxy?)streaming-opts (cond-> {:model model :input parts :tools (vals tools):credentials credentials :ai-proxy? ai-proxy?:fast? (metabot.settings/llm-fast-mode)}@@ -576,7 +597,8 @@:ai-proxy? ai-proxy?})tracking-opts (-> opts(dissoc :required-permission)- (assoc :model provider-and-model :ai-proxy? ai-proxy?))+ (assoc :model provider-and-model :provider provider :model-name model+ :ai-proxy? ai-proxy?))streaming-opts (cond-> {:model model:input input:schema json-schemadiff --git a/src/metabase/metabot/usage.clj b/src/metabase/metabot/usage.cljindex 81de97c2be0..6bcc7e4815f 100644--- a/src/metabase/metabot/usage.clj+++ b/src/metabase/metabot/usage.clj@@ -19,6 +19,8 @@[:map[:source :string][:model :string]+ [:provider {:optional true} [:maybe :string]]+ [:model-name {:optional true} [:maybe :string]][:prompt-tokens [:int {:min 0}]][:completion-tokens [:int {:min 0}]][:cache-creation-tokens {:optional true} [:maybe [:int {:min 0}]]]diff --git a/test/metabase/analytics/llm_token_usage_test.clj b/test/metabase/analytics/llm_token_usage_test.cljindex 3be227154be..eb3be216705 100644--- a/test/metabase/analytics/llm_token_usage_test.clj+++ b/test/metabase/analytics/llm_token_usage_test.clj@@ -93,9 +93,10 @@(deftest track-prometheus!-test(mt/with-prometheus-system! [_ system]- (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag"}]+ (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag" :provider "anthropic"}](testing "increments prometheus metrics with correct labels and values"(llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"+ :provider "anthropic":tag "test-tag":prompt-tokens 100:completion-tokens 50})@@ -108,6 +109,7 @@(clear-llm-metrics!)(testing "positive cache token fields increment their counters"(llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"+ :provider "anthropic":tag "test-tag":prompt-tokens 100:completion-tokens 50@@ -116,8 +118,27 @@(is (= 400.0 (mt/metric-value system :metabase-metabot/llm-cache-creation-tokens labels)))(is (= 1600.0 (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels))))(clear-llm-metrics!)+ (testing "a call with no provider is counted under the unknown provider"+ (llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"+ :tag "test-tag"+ :prompt-tokens 100+ :completion-tokens 50})+ (is (= 100.0 (mt/metric-value system :metabase-metabot/llm-input-tokens+ (assoc labels :provider "unknown")))))+ (clear-llm-metrics!)+ (testing "the provider label separates series that would otherwise share a model and source"+ (llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"+ :provider "openrouter"+ :tag "test-tag"+ :prompt-tokens 100+ :completion-tokens 50})+ (is (zero? (mt/metric-value system :metabase-metabot/llm-input-tokens labels)))+ (is (= 100.0 (mt/metric-value system :metabase-metabot/llm-input-tokens+ (assoc labels :provider "openrouter")))))+ (clear-llm-metrics!)(testing "zero / nil cache token fields do not increment their counters"(llm-token-usage/track-prometheus! {:model-id "anthropic/claude-haiku-4-5"+ :provider "anthropic":tag "test-tag":prompt-tokens 100:completion-tokens 50@@ -139,6 +160,7 @@:prometheus true:request-id "req-123":model-id "anthropic/claude-haiku-4-5"+ :provider "anthropic":tag "test-tag":prompt-tokens 100:completion-tokens 50@@ -151,7 +173,7 @@"prompt_tokens" 100}}](snowplow-test/pop-event-data-and-user-id!))))(testing "Prometheus metrics incremented"- (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag"}]+ (let [labels {:model "anthropic/claude-haiku-4-5" :source "test-tag" :provider "anthropic"}](is (= 100.0 (mt/metric-value system :metabase-metabot/llm-input-tokens labels)))(is (= 50.0 (mt/metric-value system :metabase-metabot/llm-output-tokens labels)))(is (= 150.0 (:sum (mt/metric-value system :metabase-metabot/llm-tokens-per-call labels)))))))))@@ -171,7 +193,7 @@(is (empty? (snowplow-test/pop-event-data-and-user-id!))))(testing "Prometheus still fires"(is (= 100.0 (mt/metric-value system :metabase-metabot/llm-input-tokens- {:model "anthropic/claude-haiku-4-5" :source "test-tag"}))))))+ {:model "anthropic/claude-haiku-4-5" :source "test-tag" :provider "unknown"}))))))(clear-llm-metrics!)(testing "Prometheus suppressed when :prometheus false"(mt/with-temporary-setting-values [premium-embedding-token nil@@ -190,7 +212,7 @@(snowplow-test/pop-event-data-and-user-id!))))(testing "no Prometheus metrics incremented"(is (= 0.0 (mt/metric-value system :metabase-metabot/llm-input-tokens- {:model "openai/gpt-4" :source "none"})))))))))+ {:model "openai/gpt-4" :source "none" :provider "unknown"})))))))))(deftest track-token-usage!-both-false-error-test(testing "throws when both :snowplow and :prometheus are false"diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 54b65f4d0e8..88b5277565d 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -914,7 +914,8 @@{:profile-id "internal"})))(is (== 2 (mt/metric-value system :metabase-metabot/llm-requests{:model "openrouter/anthropic/claude-haiku-4-5"- :source "agent"})))+ :source "agent"+ :provider "openrouter"})))(is (== 1 (:count (mt/metric-value system :metabase-metabot/agent-duration-ms{:profile-id "internal"}))))(is (pos? (:sum (mt/metric-value system :metabase-metabot/agent-duration-ms@@ -940,7 +941,8 @@{:profile-id "internal"})))(is (== 1 (mt/metric-value system :metabase-metabot/llm-requests{:model "openrouter/anthropic/claude-haiku-4-5"- :source "agent"})))+ :source "agent"+ :provider "openrouter"})))(is (== 1 (:count (mt/metric-value system :metabase-metabot/agent-duration-ms{:profile-id "internal"}))))(is (pos? (:sum (mt/metric-value system :metabase-metabot/agent-duration-msdiff --git a/test/metabase/metabot/api/document_test.clj b/test/metabase/metabot/api/document_test.cljindex 72dfc4d0ef9..ba929f2b992 100644--- a/test/metabase/metabot/api/document_test.clj+++ b/test/metabase/metabot/api/document_test.clj@@ -72,11 +72,11 @@(is (== 1 (:sum (mt/metric-value system :metabase-metabot/agent-iterations{:profile-id "document-generate-content"}))))(is (== 1 (mt/metric-value system :metabase-metabot/llm-requests- {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent" :provider "openrouter"})))(is (== 100 (mt/metric-value system :metabase-metabot/llm-input-tokens- {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent" :provider "openrouter"})))(is (== 20 (mt/metric-value system :metabase-metabot/llm-output-tokens- {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"}))))))+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent" :provider "openrouter"}))))))(deftest generate-content-tool-call-produces-draft-card-test(testing "a document_construct_sql_chart tool call round trip produces a :draft_card (#73690)"diff --git a/test/metabase/metabot/example_question_generator_test.clj b/test/metabase/metabot/example_question_generator_test.cljindex e8f5086b8a7..1bc73e17520 100644--- a/test/metabase/metabot/example_question_generator_test.clj+++ b/test/metabase/metabot/example_question_generator_test.clj@@ -151,7 +151,7 @@(mt/with-prometheus-system! [_ system](mt/with-temporary-setting-values [llm-providers llm.tu/default-connectionsllm-metabot-provider "openrouter/test-model"]- (let [labels {:model "openrouter/test-model" :source "example-question-generation"}]+ (let [labels {:model "openrouter/test-model" :source "example-question-generation" :provider "openrouter"}](testing "increments llm-requests and observes duration on success"(mt/with-dynamic-fn-redefs [openrouter/openrouter(constantly (test-util/mock-llm-responsediff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 9c58b0a8943..ab7791e20c3 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -77,6 +77,13 @@(#'self/parse-provider-model "deepseek/deepseek-v4-flash")))(is (=? {:provider "google" :model "google/gemini-3.5-flash" :ai-proxy? false}(#'self/parse-provider-model "google/google/gemini-3.5-flash"))))+ (testing "resolves the provider type, not the admin's name for the connection"+ (llm.tu/with-connections [{:key "openrouter-1"+ :type "openrouter"+ :name "openrouter-1"+ :config {:api-key "sk-or-v1-test"}}]+ (is (=? {:provider "openrouter" :model "anthropic/claude-sonnet-4.6" :ai-proxy? false}+ (#'self/parse-provider-model "openrouter-1/anthropic/claude-sonnet-4.6")))))(testing "a vLLM served model is often a Hugging Face repo id, so the model segment keeps its slashes"(llm.tu/with-connections [(llm.tu/connection "vllm")](is (=? {:provider "vllm" :model "mlx-community/Qwen3-14B-4bit" :ai-proxy? false}@@ -1374,7 +1381,7 @@(llm.tu/with-default-connections(mt/with-prometheus-system! [_ system](mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)]- (let [labels {:model "openrouter/test-model" :source "metabot_agent"}]+ (let [labels {:model "openrouter/test-model" :source "metabot_agent" :provider "openrouter"}](testing "increments llm-requests and observes duration on success"(mt/with-dynamic-fn-redefs [openrouter/openrouter (constantly (test-util/mock-llm-response [{:type :start :id "m1"}]))](run! identity (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"})))@@ -1470,13 +1477,22 @@:model "test-model"}]))](run! identity (self/call-llm "openrouter/test-model" nil [] {} {:tag "metabot_agent"})))(is (zero? (mt/metric-value system :metabase-metabot/llm-cache-creation-tokens labels)))- (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels)))))))))+ (is (zero? (mt/metric-value system :metabase-metabot/llm-cache-read-tokens labels))))+ (testing "labels a managed-proxy call with the metabase provider"+ (mt/with-dynamic-fn-redefs [self.claude/claude+ (constantly (test-util/mock-llm-response+ [{:type :start :id "m1"}+ {:type :usage :usage {:promptTokens 10 :completionTokens 5}}]))]+ (run! identity (self/call-llm "metabase/anthropic/claude-haiku-4-5" nil [] {} {:tag "metabot_agent"})))+ (let [managed-labels (assoc labels :model "metabase/anthropic/claude-haiku-4-5" :provider "metabase")]+ (is (== 1 (mt/metric-value system :metabase-metabot/llm-requests managed-labels)))+ (is (== 10 (mt/metric-value system :metabase-metabot/llm-input-tokens managed-labels))))))))))(deftest call-llm-structured-prometheus-test(llm.tu/with-default-connections(mt/with-prometheus-system! [_ system](mt/with-dynamic-fn-redefs [self/retry-delay-ms (constantly 0)]- (let [labels {:model "openrouter/test-model" :source "metabot_agent"}+ (let [labels {:model "openrouter/test-model" :source "metabot_agent" :provider "openrouter"}success-mock (test-util/mock-llm-response[{:type :start :id "m1"}{:type :tool-input :id "call-1" :function "json"@@ -1632,6 +1648,35 @@"session_id" "00000000-0000-0000-0000-000000000002"}}]token-events))))))))))+;;; ===================== Usage Log Tests =====================++(deftest call-llm-usage-log-test+ (testing "call-llm and call-llm-structured log the provider type and the model as the provider names it"+ (llm.tu/with-connections [(assoc (llm.tu/connection "openrouter") :key "openrouter-1")+ (llm.tu/connection "metabase")]+ (let [model-ref "openrouter-1/anthropic/claude-sonnet-4.6"+ response (test-util/mock-llm-response+ [{:type :start :id "m1"}+ {:type :tool-input :id "call-1" :function "json" :arguments {:answer "42"}}+ {:type :usage :usage {:promptTokens 10 :completionTokens 5}}])+ logged (atom [])]+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (constantly response)+ self.claude/claude (constantly response)+ usage/log-ai-usage! #(swap! logged conj %)]+ (run! identity (self/call-llm model-ref nil [] {} {:tag "metabot_agent"}))+ (self/call-llm-structured model-ref [{:role "user" :content "test"}]+ {:type "object" :properties {:answer {:type "string"}}} 0.3 1024+ {:tag "metabot_agent"})+ (run! identity (self/call-llm "metabase/anthropic/claude-haiku-4-5" nil [] {} {:tag "metabot_agent"})))+ (is (=? (concat (repeat 2 {:model model-ref+ :provider "openrouter"+ :model-name "anthropic/claude-sonnet-4.6"})+ [{:model "metabase/anthropic/claude-haiku-4-5"+ :provider "anthropic"+ :model-name "claude-haiku-4-5"+ :ai-proxied true}])+ @logged))))))+;;; ----- gating: usage-limit + permission checks in call-llm-structured-with-trace -----;;; (UXW-4126) The structured-with-trace path enforces usage limits unconditionally and;;; an optional `:required-permission` against the current user's metabot perms.