Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1321 BOT-1230 Switch to AiUsageLog for token usage reporting in metabot-stats (#72662)

applebyda0548360409
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 5e7fb17c3ff..62819d0eff5 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1016,9 +1016,9 @@
{:team "UX West"
:api #{metabase.internal-stats.core}
:uses #{app-db
- models
- util}
- :model-imports #{:model/Card
+ models}
+ :model-imports #{:model/AiUsageLog
+ :model/Card
:model/Dashboard
:model/MetabotMessage
:model/QueryExecution
diff --git a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj b/enterprise/backend/src/metabase_enterprise/metabot/usage.clj
index ccdffbe9a64..607416fca1a 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot/usage.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot/usage.clj
@@ -19,7 +19,7 @@
"Record an LLM API call in the ai_usage_log table."
:feature :none
[{:keys [source model prompt-tokens completion-tokens
- user-id tenant-id conversation-id profile-id request-id]}]
+ user-id tenant-id conversation-id profile-id request-id ai-proxied]}]
(when-not (= "user-intent-classification" source)
(try
(let [total-tokens (+ prompt-tokens completion-tokens)]
@@ -33,7 +33,8 @@
:tenant_id (or tenant-id (some-> api/*current-user* deref :tenant_id))
:conversation_id conversation-id
:profile_id (some-> profile-id name)
- :request_id request-id}))
+ :request_id request-id
+ :ai_proxied ai-proxied}))
(catch Exception e
(log/warn e "Failed to log LLM usage to ai_usage_log")))))
diff --git a/test/metabase/internal_stats/metabot_test.clj b/enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.clj
similarity index 60%
rename from test/metabase/internal_stats/metabot_test.clj
rename to enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.clj
index 7de3b275f19..1e93fdfacc0 100644
--- a/test/metabase/internal_stats/metabot_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/internal_stats/metabot_test.clj
@@ -1,8 +1,9 @@
-(ns metabase.internal-stats.metabot-test
+(ns metabase-enterprise.internal-stats.metabot-test
(:require
- [clojure.test :refer [deftest is testing]]
+ [clojure.test :refer [deftest is testing use-fixtures]]
[java-time.api :as t]
[metabase.internal-stats.metabot :as sut]
+ [metabase.metabot.example-question-generator :as eqg]
[metabase.metabot.self.claude :as claude]
[metabase.metabot.self.openai :as openai]
[metabase.metabot.self.openrouter :as openrouter]
@@ -10,8 +11,11 @@
[metabase.metabot.test-util :as mut]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[toucan2.core :as t2]))
+(use-fixtures :once (fixtures/initialize :db))
+
;; ---------------------------------------------------------------------------
;; Helpers
;; ---------------------------------------------------------------------------
@@ -39,13 +43,16 @@
:state {}}))))
(defn- backdate-messages!
- "Update created_at on all messages for a conversation to the given timestamp."
+ "Update created_at on all messages and usage log rows for a conversation to the given timestamp."
[conversation-id created-at]
(t2/update! :model/MetabotMessage {:conversation_id conversation-id}
+ {:created_at created-at})
+ (t2/update! :model/AiUsageLog {:conversation_id conversation-id}
{:created_at created-at}))
(defn- cleanup! [& conv-ids]
(doseq [cid conv-ids]
+ (t2/delete! :model/AiUsageLog :conversation_id cid)
(t2/delete! :model/MetabotMessage :conversation_id cid)
(t2/delete! :model/MetabotConversation :id cid)))
@@ -107,11 +114,21 @@
(is (= 2 (count msgs)) "should have user + assistant messages")
(is (every? true? (map :ai_proxied msgs)))))
+ (testing "ai-proxy usage log has ai_proxied = true"
+ (let [logs (t2/select :model/AiUsageLog :conversation_id conv-1)]
+ (is (= 1 (count logs)) "should have one usage log row per LLM call")
+ (is (every? true? (map :ai_proxied logs)))))
+
(testing "BYOK messages have ai_proxied = false on ALL rows"
(let [msgs (t2/select :model/MetabotMessage :conversation_id conv-5)]
(is (= 2 (count msgs)))
(is (every? false? (map :ai_proxied msgs)))))
+ (testing "BYOK usage log has ai_proxied = false"
+ (let [logs (t2/select :model/AiUsageLog :conversation_id conv-5)]
+ (is (= 1 (count logs)) "should have one usage log row per LLM call")
+ (is (every? false? (map :ai_proxied logs)))))
+
(testing "usage keys are provider/model (metabase/ prefix stripped)"
;; accumulate-usage-xf strips metabase/ prefix → "openrouter/anthropic/claude-haiku-4-5"
;; JSON roundtrip keywordizes → :openrouter/anthropic/claude-haiku-4-5
@@ -240,3 +257,135 @@
(:metabot-usage stats))))
(finally
(cleanup! conv-1 conv-2 conv-3)))))))
+
+;; ---------------------------------------------------------------------------
+;; Example question generation usage tracking
+;; ---------------------------------------------------------------------------
+
+(defn- generate-example-questions!
+ "Call the example question generator with a mocked LLM that returns structured output
+ with the given model/usage."
+ [tables model prompt-tokens completion-tokens]
+ (let [mock-fn (fn [_]
+ (mut/mock-llm-response
+ [{:type :start :id "msg-eqg"}
+ {:type :tool-input
+ :id "tool-1"
+ :function "json"
+ :arguments {:questions ["What is the total?" "Show me trends"]}}
+ {:type :usage
+ :model model
+ :usage {:promptTokens prompt-tokens :completionTokens completion-tokens}
+ :id "msg-eqg"}]))]
+ (with-redefs [openrouter/openrouter mock-fn
+ claude/claude mock-fn
+ openai/openai mock-fn]
+ (eqg/generate-example-questions {:tables tables :metrics []}))))
+
+(defn- max-usage-log-id
+ "Return the current max id in ai_usage_log, or 0 if empty."
+ []
+ (or (:max (t2/query-one {:select [[:%max.id :max]] :from [:ai_usage_log]})) 0))
+
+(defn- cleanup-usage-logs-after! [min-id]
+ (t2/delete! :model/AiUsageLog :id [:> min-id]))
+
+;; NOTE: generate-example-questions! uses `future` internally (process-batch-parallel),
+;; so ai_usage_log inserts happen on separate threads outside any with-transaction scope.
+;; We snapshot max(id) before each test and only delete rows inserted after that.
+
+(deftest example-question-generation-creates-usage-log-test
+ (search.tu/with-index-disabled
+ (let [baseline (max-usage-log-id)
+ clock (t/mock-clock (t/instant "2026-04-01T12:00:00Z") "UTC")
+ yesterday (t/offset-date-time 2026 3 31 10 0 0 0 (t/zone-offset "+00"))
+ today (t/offset-date-time 2026 4 1 9 0 0 0 (t/zone-offset "+00"))
+ model "anthropic/claude-haiku-4-5"
+ tables [{:name "Orders" :fields [{:name "id"} {:name "total"}]}]]
+ (t/with-clock clock
+ (try
+ (mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider
+ "metabase/openrouter/anthropic/claude-haiku-4-5"]
+ ;; Yesterday's generation
+ (generate-example-questions! tables model 400 100)
+
+ (testing "ai_usage_log row is created with ai_proxied = true"
+ (is (=? [{:ai_proxied true
+ :total_tokens 500}]
+ (t2/select :model/AiUsageLog :id [:> baseline]))))
+
+ ;; backdate so it lands in yesterday's window
+ (t2/update! :model/AiUsageLog {:id [:> baseline]}
+ {:created_at yesterday})
+
+ ;; Today's generation — exercises rolling usage
+ (let [before-today (max-usage-log-id)]
+ (generate-example-questions! tables model 200 60)
+ (t2/update! :model/AiUsageLog {:id [:> before-today]}
+ {:created_at today})))
+
+ (testing "metabot-stats includes yesterday totals and today's rolling usage"
+ (is (=? {:metabot-tokens 500
+ :metabot-usage {"openrouter:anthropic/claude-haiku-4-5:tokens" 500}
+ :metabot-rolling-usage {"openrouter:anthropic/claude-haiku-4-5:tokens" 260}
+ :metabot-rolling-usage-date "2026-04-01"}
+ (sut/metabot-stats))))
+ (finally
+ (cleanup-usage-logs-after! baseline)))))))
+
+(deftest example-question-generation-byok-not-in-stats-test
+ (search.tu/with-index-disabled
+ (let [baseline (max-usage-log-id)
+ clock (t/mock-clock (t/instant "2026-04-01T12:00:00Z") "UTC")
+ yesterday (t/offset-date-time 2026 3 31 10 0 0 0 (t/zone-offset "+00"))
+ model "anthropic/claude-haiku-4-5"
+ tables [{:name "Orders" :fields [{:name "id"} {:name "total"}]}]]
+ (t/with-clock clock
+ (try
+ (mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider
+ "openrouter/anthropic/claude-haiku-4-5"]
+ (generate-example-questions! tables model 400 100)
+
+ (testing "ai_usage_log row is created with ai_proxied = false for BYOK"
+ (is (=? [{:ai_proxied false}]
+ (t2/select :model/AiUsageLog :id [:> baseline]))))
+
+ (t2/update! :model/AiUsageLog {:id [:> baseline]}
+ {:created_at yesterday}))
+
+ (testing "BYOK example question usage does not appear in metabot-stats"
+ (is (nil? (sut/metabot-stats))))
+ (finally
+ (cleanup-usage-logs-after! baseline)))))))
+
+(deftest example-question-generation-combined-with-chat-test
+ (search.tu/with-index-disabled
+ (let [baseline (max-usage-log-id)
+ clock (t/mock-clock (t/instant "2026-04-01T12:00:00Z") "UTC")
+ yesterday (t/offset-date-time 2026 3 31 10 0 0 0 (t/zone-offset "+00"))
+ model "anthropic/claude-haiku-4-5"
+ tables [{:name "Orders" :fields [{:name "id"} {:name "total"}]}]
+ conv-id (str (random-uuid))]
+ (t/with-clock clock
+ (try
+ (mt/with-temporary-setting-values [metabot.settings/llm-metabot-provider
+ "metabase/openrouter/anthropic/claude-haiku-4-5"]
+ ;; Chat conversation
+ (send-message! conv-id "What is 2+2?" model 200 50)
+ (backdate-messages! conv-id yesterday)
+
+ ;; Example question generation (no conversation)
+ (generate-example-questions! tables model 300 100)
+ (t2/update! :model/AiUsageLog {:id [:> baseline]
+ :source "example-question-generation"}
+ {:created_at yesterday}))
+
+ (testing "metabot-stats includes both chat and example question generation"
+ ;; chat: 250, eqg: 400 → total 650
+ (is (=? {:metabot-tokens 650
+ :metabot-queries 1
+ :metabot-usage {"openrouter:anthropic/claude-haiku-4-5:tokens" 650}}
+ (sut/metabot-stats))))
+ (finally
+ (cleanup-usage-logs-after! baseline)
+ (cleanup! conv-id)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj
index d861fd7022e..03608159537 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot/usage_test.clj
@@ -36,7 +36,8 @@
{:source "log-test"
:model "anthropic/claude-test"
:prompt-tokens 100
- :completion-tokens 50})
+ :completion-tokens 50
+ :ai-proxied true})
(try
(is (= (inc before-count)
(t2/count :model/AiUsageLog :user_id user-id :source "log-test")))
@@ -46,7 +47,8 @@
(is (= 100 (:prompt_tokens row)))
(is (= 50 (:completion_tokens row)))
(is (= 150 (:total_tokens row)))
- (is (= user-id (:user_id row))))
+ (is (= user-id (:user_id row)))
+ (is (true? (:ai_proxied row))))
(finally
(t2/delete! :model/AiUsageLog :user_id user-id :source "log-test")))))))))
@@ -118,6 +120,22 @@
(finally
(t2/delete! :model/AiUsageLog :source "explicit-user-test"))))))))
+(deftest log-ai-usage!-ai-proxied-defaults-to-nil-test
+ (mt/with-premium-features #{:ai-controls}
+ (testing "log-ai-usage! stores nil for ai_proxied when not provided"
+ (mt/with-test-user :rasta
+ (usage/log-ai-usage!
+ {:source "proxied-default-test"
+ :model "test/model"
+ :prompt-tokens 1
+ :completion-tokens 1})
+ (try
+ (let [row (t2/select-one :model/AiUsageLog :source "proxied-default-test"
+ {:order-by [[:id :desc]]})]
+ (is (nil? (:ai_proxied row))))
+ (finally
+ (t2/delete! :model/AiUsageLog :source "proxied-default-test")))))))
+
;;; ------------------------------------------ check-usage-limits! ------------------------------------------
(deftest no-limits-configured-returns-nil-test
diff --git a/src/metabase/internal_stats/metabot.clj b/src/metabase/internal_stats/metabot.clj
index 2ae2d137310..6d256bea5f3 100644
--- a/src/metabase/internal_stats/metabot.clj
+++ b/src/metabase/internal_stats/metabot.clj
@@ -2,37 +2,36 @@
(:require
[clojure.string :as str]
[java-time.api :as t]
- [metabase.util :as u]
[toucan2.core :as t2]))
(defn- usage-by-model
"Aggregate combined tokens by provider:model for a given UTC date."
[date-utc]
- (let [usages (t2/select-fn-vec :usage
- [:model/MetabotMessage :usage]
- {:where [:and
- :ai_proxied
- [:= [:cast :created_at :date] [:cast date-utc :date]]
- [:not= :usage nil]]})]
- (->> (for [usage usages
- [prov-model tokens] usage
- :let [k (str/replace-first (u/qualified-name prov-model) "/" ":")]]
- {(str k ":tokens") (+ (:prompt tokens) (:completion tokens))})
+ (let [rows (t2/select [:model/AiUsageLog :model [:%sum.total_tokens :tokens]]
+ {:where [:and
+ :ai_proxied
+ [:= [:cast :created_at :date] [:cast date-utc :date]]]
+ :group-by [:model]})]
+ (->> (for [{:keys [model tokens]} rows
+ :let [k (-> model
+ (str/replace-first "metabase/" "")
+ (str/replace-first "/" ":"))]]
+ {(str k ":tokens") (long tokens)})
(apply merge-with +)
not-empty)))
(defn metabot-stats
- "Calculate total Metabot token usage over a window of the previous UTC day 00:00-23:59,
- plus rolling usage stats for today."
+ "Calculate total Metabot token usage over a window of the previous UTC day 00:00-23:59 plus rolling usage for today.
+
+ Note that the AiUsageLog table is only populated for EE builds, so this will never return data in OSS."
[]
(let [today-utc (t/offset-date-time (t/zone-offset "+00"))
yesterday-utc (t/minus today-utc (t/days 1))
tokens (or (t2/select-one-fn :sum
- [:model/MetabotMessage [:%sum.total_tokens :sum]]
+ [:model/AiUsageLog [:%sum.total_tokens :sum]]
{:where [:and
:ai_proxied
- [:= [:cast :created_at :date] [:cast yesterday-utc :date]]
- [:not= :usage nil]]})
+ [:= [:cast :created_at :date] [:cast yesterday-utc :date]]]})
0)
rolling-usage (usage-by-model today-utc)]
(when (or (pos? tokens) (seq rolling-usage))
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 6080dbc301c..48963864e44 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -156,7 +156,7 @@
- `:session-id` — conversation UUID string
- `:source` — the source of the request (e.g., 'metabot_agent', 'document_generate_content').
Indicates which API endpoint or workflow initiated the LLM call."
- [{:keys [model profile-id request-id session-id source tag]}]
+ [{:keys [model profile-id request-id session-id source tag ai-proxy?]}]
(let [start-ms (u/start-timer)]
(map (fn [part]
(when (= (:type part) :usage)
@@ -187,7 +187,8 @@
:completion-tokens completion
:conversation-id session-id
:profile-id profile-id
- :request-id request-id})))
+ :request-id request-id
+ :ai-proxied (boolean ai-proxy?)})))
part))))
(defn- report-tool-usage-xf
@@ -276,7 +277,7 @@
(let [{:keys [provider stream-fn model ai-proxy?]} (parse-provider-model provider-and-model)]
(log/info "Calling LLM" {:provider provider :model model :parts (count parts) :tools (count tools)
:tool-choice tool-choice :ai-proxy? ai-proxy?})
- (let [tracking-opts (assoc tracking-opts :model provider-and-model)
+ (let [tracking-opts (assoc tracking-opts :model provider-and-model :ai-proxy? ai-proxy?)
streaming-opts (cond-> {:model model :input parts :tools (vals tools) :ai-proxy? ai-proxy?}
system-msg (assoc :system system-msg)
(and (seq tools)
@@ -318,8 +319,11 @@
Returns the parsed JSON map from the forced tool call."
[provider-and-model messages json-schema temperature max-tokens tracking-opts]
(let [{:keys [provider stream-fn model ai-proxy?]} (parse-provider-model provider-and-model)
- _ (log/info "Calling LLM (structured)" {:provider provider :model model :msg-count (count messages)})
- tracking-opts (assoc tracking-opts :model provider-and-model)
+ _ (log/info "Calling LLM (structured)" {:provider provider
+ :model model
+ :msg-count (count messages)
+ :ai-proxy? ai-proxy?})
+ tracking-opts (assoc tracking-opts :model provider-and-model :ai-proxy? ai-proxy?)
streaming-opts {:model model
:input messages
:schema json-schema
diff --git a/src/metabase/metabot/usage.clj b/src/metabase/metabot/usage.clj
index e2887b810db..dfd0ed8fc04 100644
--- a/src/metabase/metabot/usage.clj
+++ b/src/metabase/metabot/usage.clj
@@ -19,7 +19,8 @@
[:tenant-id {:optional true} [:maybe ms/PositiveInt]]
[:conversation-id {:optional true} [:maybe :string]]
[:profile-id {:optional true} [:maybe :keyword]]
- [:request-id {:optional true} [:maybe :string]]])
+ [:request-id {:optional true} [:maybe :string]]
+ [:ai-proxied {:optional true} [:maybe :boolean]]])
(defenterprise-schema log-ai-usage! :- :any
"Record an LLM API call in the ai_usage_log table.