The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-1321 BOT-1230 Switch to AiUsageLog for token usage reporting in metabot-stats (#72662)
applebyda0548360409
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 5e7fb17c3ff..62819d0eff5 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1016,9 +1016,9 @@{:team "UX West":api #{metabase.internal-stats.core}:uses #{app-db- models- util}- :model-imports #{:model/Card+ models}+ :model-imports #{:model/AiUsageLog+ :model/Card:model/Dashboard:model/MetabotMessage:model/QueryExecutiondiff --git a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj b/enterprise/backend/src/metabase_enterprise/metabot/usage.cljindex ccdffbe9a64..607416fca1a 100644--- a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot/usage.clj@@ -19,7 +19,7 @@"Record an LLM API call in the ai_usage_log table.":feature :none[{:keys [source model prompt-tokens completion-tokens- user-id tenant-id conversation-id profile-id request-id]}]+ user-id tenant-id conversation-id profile-id request-id ai-proxied]}](when-not (= "user-intent-classification" source)(try(let [total-tokens (+ prompt-tokens completion-tokens)]@@ -33,7 +33,8 @@:tenant_id (or tenant-id (some-> api/*current-user* deref :tenant_id)):conversation_id conversation-id:profile_id (some-> profile-id name)- :request_id request-id}))+ :request_id request-id+ :ai_proxied ai-proxied}))(catch Exception e(log/warn e "Failed to log LLM usage to ai_usage_log")))))diff --git a/test/metabase/internal_stats/metabot_test.clj b/enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.cljsimilarity index 60%rename from test/metabase/internal_stats/metabot_test.cljrename to enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.cljindex 7de3b275f19..1e93fdfacc0 100644--- a/test/metabase/internal_stats/metabot_test.clj+++ b/enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.clj@@ -1,8 +1,9 @@-(ns metabase.internal-stats.metabot-test+(ns metabase-enterprise.internal-stats.metabot-test(:require- [clojure.test :refer [deftest is testing]]+ [clojure.test :refer [deftest is testing use-fixtures]][java-time.api :as t][metabase.internal-stats.metabot :as sut]+ [metabase.metabot.example-question-generator :as eqg][metabase.metabot.self.claude :as claude][metabase.metabot.self.openai :as openai][metabase.metabot.self.openrouter :as openrouter]@@ -10,8 +11,11 @@[metabase.metabot.test-util :as mut][metabase.search.test-util :as search.tu][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][toucan2.core :as t2]))+(use-fixtures :once (fixtures/initialize :db))+;; ---------------------------------------------------------------------------;; Helpers;; ---------------------------------------------------------------------------@@ -39,13 +43,16 @@:state {}}))))(defn- backdate-messages!- "Update created_at on all messages for a conversation to the given timestamp."+ "Update created_at on all messages and usage log rows for a conversation to the given timestamp."[conversation-id created-at](t2/update! :model/MetabotMessage {:conversation_id conversation-id}+ {:created_at created-at})+ (t2/update! :model/AiUsageLog {:conversation_id conversation-id}{:created_at created-at}))(defn- cleanup! [& conv-ids](doseq [cid conv-ids]+ (t2/delete! :model/AiUsageLog :conversation_id cid)(t2/delete! :model/MetabotMessage :conversation_id cid)(t2/delete! :model/MetabotConversation :id cid)))@@ -107,11 +114,21 @@(is (= 2 (count msgs)) "should have user + assistant messages")(is (every? true? (map :ai_proxied msgs)))))+ (testing "ai-proxy usage log has ai_proxied = true"+ (let [logs (t2/select :model/AiUsageLog :conversation_id conv-1)]+ (is (= 1 (count logs)) "should have one usage log row per LLM call")+ (is (every? true? (map :ai_proxied logs)))))+(testing "BYOK messages have ai_proxied = false on ALL rows"(let [msgs (t2/select :model/MetabotMessage :conversation_id conv-5)](is (= 2 (count msgs)))(is (every? false? (map :ai_proxied msgs)))))+ (testing "BYOK usage log has ai_proxied = false"+ (let [logs (t2/select :model/AiUsageLog :conversation_id conv-5)]+ (is (= 1 (count logs)) "should have one usage log row per LLM call")+ (is (every? false? (map :ai_proxied logs)))))+(testing "usage keys are provider/model (metabase/ prefix stripped)";; accumulate-usage-xf strips metabase/ prefix → "openrouter/anthropic/claude-haiku-4-5";; JSON roundtrip keywordizes → :openrouter/anthropic/claude-haiku-4-5@@ -240,3 +257,135 @@(:metabot-usage stats))))(finally(cleanup! conv-1 conv-2 conv-3)))))))++;; ---------------------------------------------------------------------------+;; Example question generation usage tracking+;; ---------------------------------------------------------------------------++(defn- generate-example-questions!+ "Call the example question generator with a mocked LLM that returns structured output+ with the given model/usage."+ [tables model prompt-tokens completion-tokens]+ (let [mock-fn (fn [_]+ (mut/mock-llm-response+ [{:type :start :id "msg-eqg"}+ {:type :tool-input+ :id "tool-1"+ :function "json"+ :arguments {:questions ["What is the total?" "Show me trends"]}}+ {:type :usage+ :model model+ :usage {:promptTokens prompt-tokens :completionTokens completion-tokens}+ :id "msg-eqg"}]))]+ (with-redefs [openrouter/openrouter mock-fn+ claude/claude mock-fn+ openai/openai mock-fn]+ (eqg/generate-example-questions {:tables tables :metrics []}))))++(defn- max-usage-log-id+ "Return the current max id in ai_usage_log, or 0 if empty."+ []+ (or (:max (t2/query-one {:select [[:%max.id :max]] :from [:ai_usage_log]})) 0))++(defn- cleanup-usage-logs-after! [min-id]+ (t2/delete! :model/AiUsageLog :id [:> min-id]))++;; NOTE: generate-example-questions! uses `future` internally (process-batch-parallel),+;; so ai_usage_log inserts happen on separate threads outside any with-transaction scope.+;; We snapshot max(id) before each test and only delete rows inserted after that.++(deftest example-question-generation-creates-usage-log-test+ (search.tu/with-index-disabled+ (let [baseline (max-usage-log-id)+ clock (t/mock-clock (t/instant "2026-04-01T12:00:00Z") "UTC")+ yesterday (t/offset-date-time 2026 3 31 10 0 0 0 (t/zone-offset "+00"))+ today (t/offset-date-time 2026 4 1 9 0 0 0 (t/zone-offset "+00"))+ model "anthropic/claude-haiku-4-5"+ tables [{:name "Orders" :fields [{:name "id"} {:name "total"}]}]]+ (t/with-clock clock+ (try+ (mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider+ "metabase/openrouter/anthropic/claude-haiku-4-5"]+ ;; Yesterday's generation+ (generate-example-questions! tables model 400 100)++ (testing "ai_usage_log row is created with ai_proxied = true"+ (is (=? [{:ai_proxied true+ :total_tokens 500}]+ (t2/select :model/AiUsageLog :id [:> baseline]))))++ ;; backdate so it lands in yesterday's window+ (t2/update! :model/AiUsageLog {:id [:> baseline]}+ {:created_at yesterday})++ ;; Today's generation — exercises rolling usage+ (let [before-today (max-usage-log-id)]+ (generate-example-questions! tables model 200 60)+ (t2/update! :model/AiUsageLog {:id [:> before-today]}+ {:created_at today})))++ (testing "metabot-stats includes yesterday totals and today's rolling usage"+ (is (=? {:metabot-tokens 500+ :metabot-usage {"openrouter:anthropic/claude-haiku-4-5:tokens" 500}+ :metabot-rolling-usage {"openrouter:anthropic/claude-haiku-4-5:tokens" 260}+ :metabot-rolling-usage-date "2026-04-01"}+ (sut/metabot-stats))))+ (finally+ (cleanup-usage-logs-after! baseline)))))))++(deftest example-question-generation-byok-not-in-stats-test+ (search.tu/with-index-disabled+ (let [baseline (max-usage-log-id)+ clock (t/mock-clock (t/instant "2026-04-01T12:00:00Z") "UTC")+ yesterday (t/offset-date-time 2026 3 31 10 0 0 0 (t/zone-offset "+00"))+ model "anthropic/claude-haiku-4-5"+ tables [{:name "Orders" :fields [{:name "id"} {:name "total"}]}]]+ (t/with-clock clock+ (try+ (mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider+ "openrouter/anthropic/claude-haiku-4-5"]+ (generate-example-questions! tables model 400 100)++ (testing "ai_usage_log row is created with ai_proxied = false for BYOK"+ (is (=? [{:ai_proxied false}]+ (t2/select :model/AiUsageLog :id [:> baseline]))))++ (t2/update! :model/AiUsageLog {:id [:> baseline]}+ {:created_at yesterday}))++ (testing "BYOK example question usage does not appear in metabot-stats"+ (is (nil? (sut/metabot-stats))))+ (finally+ (cleanup-usage-logs-after! baseline)))))))++(deftest example-question-generation-combined-with-chat-test+ (search.tu/with-index-disabled+ (let [baseline (max-usage-log-id)+ clock (t/mock-clock (t/instant "2026-04-01T12:00:00Z") "UTC")+ yesterday (t/offset-date-time 2026 3 31 10 0 0 0 (t/zone-offset "+00"))+ model "anthropic/claude-haiku-4-5"+ tables [{:name "Orders" :fields [{:name "id"} {:name "total"}]}]+ conv-id (str (random-uuid))]+ (t/with-clock clock+ (try+ (mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider+ "metabase/openrouter/anthropic/claude-haiku-4-5"]+ ;; Chat conversation+ (send-message! conv-id "What is 2+2?" model 200 50)+ (backdate-messages! conv-id yesterday)++ ;; Example question generation (no conversation)+ (generate-example-questions! tables model 300 100)+ (t2/update! :model/AiUsageLog {:id [:> baseline]+ :source "example-question-generation"}+ {:created_at yesterday}))++ (testing "metabot-stats includes both chat and example question generation"+ ;; chat: 250, eqg: 400 → total 650+ (is (=? {:metabot-tokens 650+ :metabot-queries 1+ :metabot-usage {"openrouter:anthropic/claude-haiku-4-5:tokens" 650}}+ (sut/metabot-stats))))+ (finally+ (cleanup-usage-logs-after! baseline)+ (cleanup! conv-id)))))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.cljindex d861fd7022e..03608159537 100644--- a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj@@ -36,7 +36,8 @@{:source "log-test":model "anthropic/claude-test":prompt-tokens 100- :completion-tokens 50})+ :completion-tokens 50+ :ai-proxied true})(try(is (= (inc before-count)(t2/count :model/AiUsageLog :user_id user-id :source "log-test")))@@ -46,7 +47,8 @@(is (= 100 (:prompt_tokens row)))(is (= 50 (:completion_tokens row)))(is (= 150 (:total_tokens row)))- (is (= user-id (:user_id row))))+ (is (= user-id (:user_id row)))+ (is (true? (:ai_proxied row))))(finally(t2/delete! :model/AiUsageLog :user_id user-id :source "log-test")))))))))@@ -118,6 +120,22 @@(finally(t2/delete! :model/AiUsageLog :source "explicit-user-test"))))))))+(deftest log-ai-usage!-ai-proxied-defaults-to-nil-test+ (mt/with-premium-features #{:ai-controls}+ (testing "log-ai-usage! stores nil for ai_proxied when not provided"+ (mt/with-test-user :rasta+ (usage/log-ai-usage!+ {:source "proxied-default-test"+ :model "test/model"+ :prompt-tokens 1+ :completion-tokens 1})+ (try+ (let [row (t2/select-one :model/AiUsageLog :source "proxied-default-test"+ {:order-by [[:id :desc]]})]+ (is (nil? (:ai_proxied row))))+ (finally+ (t2/delete! :model/AiUsageLog :source "proxied-default-test")))))))+;;; ------------------------------------------ check-usage-limits! ------------------------------------------(deftest no-limits-configured-returns-nil-testdiff --git a/src/metabase/internal_stats/metabot.clj b/src/metabase/internal_stats/metabot.cljindex 2ae2d137310..6d256bea5f3 100644--- a/src/metabase/internal_stats/metabot.clj+++ b/src/metabase/internal_stats/metabot.clj@@ -2,37 +2,36 @@(:require[clojure.string :as str][java-time.api :as t]- [metabase.util :as u][toucan2.core :as t2]))(defn- usage-by-model"Aggregate combined tokens by provider:model for a given UTC date."[date-utc]- (let [usages (t2/select-fn-vec :usage- [:model/MetabotMessage :usage]- {:where [:and- :ai_proxied- [:= [:cast :created_at :date] [:cast date-utc :date]]- [:not= :usage nil]]})]- (->> (for [usage usages- [prov-model tokens] usage- :let [k (str/replace-first (u/qualified-name prov-model) "/" ":")]]- {(str k ":tokens") (+ (:prompt tokens) (:completion tokens))})+ (let [rows (t2/select [:model/AiUsageLog :model [:%sum.total_tokens :tokens]]+ {:where [:and+ :ai_proxied+ [:= [:cast :created_at :date] [:cast date-utc :date]]]+ :group-by [:model]})]+ (->> (for [{:keys [model tokens]} rows+ :let [k (-> model+ (str/replace-first "metabase/" "")+ (str/replace-first "/" ":"))]]+ {(str k ":tokens") (long tokens)})(apply merge-with +)not-empty)))(defn metabot-stats- "Calculate total Metabot token usage over a window of the previous UTC day 00:00-23:59,- plus rolling usage stats for today."+ "Calculate total Metabot token usage over a window of the previous UTC day 00:00-23:59 plus rolling usage for today.++ Note that the AiUsageLog table is only populated for EE builds, so this will never return data in OSS."[](let [today-utc (t/offset-date-time (t/zone-offset "+00"))yesterday-utc (t/minus today-utc (t/days 1))tokens (or (t2/select-one-fn :sum- [:model/MetabotMessage [:%sum.total_tokens :sum]]+ [:model/AiUsageLog [:%sum.total_tokens :sum]]{:where [:and:ai_proxied- [:= [:cast :created_at :date] [:cast yesterday-utc :date]]- [:not= :usage nil]]})+ [:= [:cast :created_at :date] [:cast yesterday-utc :date]]]})0)rolling-usage (usage-by-model today-utc)](when (or (pos? tokens) (seq rolling-usage))diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 6080dbc301c..48963864e44 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -156,7 +156,7 @@- `:session-id` — conversation UUID string- `:source` — the source of the request (e.g., 'metabot_agent', 'document_generate_content').Indicates which API endpoint or workflow initiated the LLM call."- [{:keys [model profile-id request-id session-id source tag]}]+ [{:keys [model profile-id request-id session-id source tag ai-proxy?]}](let [start-ms (u/start-timer)](map (fn [part](when (= (:type part) :usage)@@ -187,7 +187,8 @@:completion-tokens completion:conversation-id session-id:profile-id profile-id- :request-id request-id})))+ :request-id request-id+ :ai-proxied (boolean ai-proxy?)})))part))))(defn- report-tool-usage-xf@@ -276,7 +277,7 @@(let [{:keys [provider stream-fn model ai-proxy?]} (parse-provider-model provider-and-model)](log/info "Calling LLM" {:provider provider :model model :parts (count parts) :tools (count tools):tool-choice tool-choice :ai-proxy? ai-proxy?})- (let [tracking-opts (assoc tracking-opts :model provider-and-model)+ (let [tracking-opts (assoc tracking-opts :model provider-and-model :ai-proxy? ai-proxy?)streaming-opts (cond-> {:model model :input parts :tools (vals tools) :ai-proxy? ai-proxy?}system-msg (assoc :system system-msg)(and (seq tools)@@ -318,8 +319,11 @@Returns the parsed JSON map from the forced tool call."[provider-and-model messages json-schema temperature max-tokens tracking-opts](let [{:keys [provider stream-fn model ai-proxy?]} (parse-provider-model provider-and-model)- _ (log/info "Calling LLM (structured)" {:provider provider :model model :msg-count (count messages)})- tracking-opts (assoc tracking-opts :model provider-and-model)+ _ (log/info "Calling LLM (structured)" {:provider provider+ :model model+ :msg-count (count messages)+ :ai-proxy? ai-proxy?})+ tracking-opts (assoc tracking-opts :model provider-and-model :ai-proxy? ai-proxy?)streaming-opts {:model model:input messages:schema json-schemadiff --git a/src/metabase/metabot/usage.clj b/src/metabase/metabot/usage.cljindex e2887b810db..dfd0ed8fc04 100644--- a/src/metabase/metabot/usage.clj+++ b/src/metabase/metabot/usage.clj@@ -19,7 +19,8 @@[:tenant-id {:optional true} [:maybe ms/PositiveInt]][:conversation-id {:optional true} [:maybe :string]][:profile-id {:optional true} [:maybe :keyword]]- [:request-id {:optional true} [:maybe :string]]])+ [:request-id {:optional true} [:maybe :string]]+ [:ai-proxied {:optional true} [:maybe :boolean]]])(defenterprise-schema log-ai-usage! :- :any"Record an LLM API call in the ai_usage_log table.