Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Send the provider and model name on token usage events (#83244)

Andrei Simionescue979237359aa
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
index 79d9dbcd388..0e9f2772f6c 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
@@ -537,6 +537,8 @@
:prometheus false ; already tracked via inc! above
:request-id (analytics.core/uuid->ai-service-hex-uuid (random-uuid))
:model-id model-name
+ :provider provider
+ :model-name model-name
:total-tokens total-tokens
:prompt-tokens prompt-tokens
:completion-tokens 0 ; embedding models don't produce completion tokens
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
index 5e5d484dde7..abc2a38f088 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
@@ -630,10 +630,15 @@
;; triggered by instance-creation setting being read for the first time)
(filter #(get-in % [:data "tag"])))]
(is (=? [{:data {"model_id" "test-model"
+ "provider" "ai-service"
+ "model_name" "test-model"
"total_tokens" 5
"prompt_tokens" 5
"tag" "embedding_generation"}}]
- events))))))))
+ events))
+ (testing "with no cache counts and a zero cost"
+ (is (=? [{:data {"cache_creation_tokens" nil "cache_read_tokens" nil "estimated_costs_usd" 0.0}}]
+ events)))))))))
(deftest test-embedding-service-snowplow-suppression
(testing "ai-service fires no token_usage event when the caller passes :snowplow? false"
diff --git a/snowplow/iglu-client-embedded/schemas/com.metabase/token_usage/jsonschema/1-0-6 b/snowplow/iglu-client-embedded/schemas/com.metabase/token_usage/jsonschema/1-0-6
new file mode 100644
index 00000000000..3bc6a01d624
--- /dev/null
+++ b/snowplow/iglu-client-embedded/schemas/com.metabase/token_usage/jsonschema/1-0-6
@@ -0,0 +1,146 @@
+{
+ "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
+ "self": {
+ "vendor": "com.metabase",
+ "name": "token_usage",
+ "format": "jsonschema",
+ "version": "1-0-6"
+ },
+ "type": "object",
+ "description": "Event to capture the token usage of individual LLM invocations. Created for billing purposes.",
+ "properties": {
+ "hashed_metabase_license_token": {
+ "type": "string",
+ "description": "The hashed value of the Metabase license token for instances with a token. For Open Source instances, this will use the snowplow anonymous uuid in the form `oss__<anonymous_uuid>` to allow unique counts of instances.",
+ "maxLength": 255
+ },
+ "user_id": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "description": "The ID of the user who sent the request to Metabot. Embeddings always send null, including those made for a user's search.",
+ "minimum": 0,
+ "maximum": 2147483647
+ },
+ "request_id": {
+ "type": "string",
+ "description": "An ID for the request the tokens were used in. A Metabot agent turn uses one ID for all of its LLM calls, so the ID groups that turn's usage. Every other call gets an ID of its own, including each embedding batch, so a single semantic search indexing run spans many IDs.",
+ "maxLength": 255
+ },
+ "session_id": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "An identifier for the conversation or session in which the tokens were used. This can help group related requests together. Can be null for non-conversational use cases.",
+ "maxLength": 255
+ },
+ "model_id": {
+ "type": "string",
+ "description": "The model the tokens were used on: for Metabot, the connection key plus the model, e.g. 'openrouter-1/anthropic/claude-sonnet-4.6'; for embeddings, the model alone.",
+ "maxLength": 255
+ },
+ "provider": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "The type of the connection the call went through: for Metabot, the connection's provider type, e.g. 'anthropic' or 'openrouter', or 'metabase' for the Metabase AI service; 'ai-service' for semantic search embeddings, whichever OpenAI-compatible endpoint serves them. Unlike the connection key at the start of a Metabot model_id, it is never a name an admin chose.",
+ "maxLength": 255
+ },
+ "model_name": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "The model Metabase asked the provider for. For Metabot it is model_id without the connection key at its start, and for the Metabase AI service without the 'anthropic/' after the key too: e.g. 'claude-sonnet-4-6', 'anthropic/claude-sonnet-4.6' on OpenRouter, or on Azure the model family plus the deployment name, e.g. 'openai/my-deployment'. For embeddings it equals model_id.",
+ "maxLength": 255
+ },
+ "total_tokens": {
+ "type": "integer",
+ "description": "The total number of tokens",
+ "minimum": 0,
+ "maximum": 2147483647
+ },
+ "prompt_tokens": {
+ "type": "integer",
+ "description": "The number of prompt tokens",
+ "minimum": 0,
+ "maximum": 2147483647
+ },
+ "completion_tokens": {
+ "type": "integer",
+ "description": "The number of completion tokens",
+ "minimum": 0,
+ "maximum": 2147483647
+ },
+ "cache_creation_tokens": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "description": "Number of input tokens written to the provider prompt cache (Anthropic cache_creation_input_tokens). Metabot sends 0 when the provider reports no cache usage; embeddings send null.",
+ "minimum": 0,
+ "maximum": 2147483647
+ },
+ "cache_read_tokens": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "description": "Number of input tokens served from the provider prompt cache (Anthropic cache_read_input_tokens). Metabot sends 0 when the provider reports no cache usage; embeddings send null.",
+ "minimum": 0,
+ "maximum": 2147483647
+ },
+ "estimated_costs_usd": {
+ "type": "number",
+ "description": "Always 0 from the Metabase app, which does not estimate costs.",
+ "minimum": 0,
+ "maximum": 2147483647
+ },
+ "duration_ms": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "description": "The time it took, in milliseconds, to receive an answer from the LLM.",
+ "minimum": 0,
+ "maximum": 2147483647
+ },
+ "tag": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "A classifier indicating the specific purpose or functionality for which the tokens were used.",
+ "maxLength": 255
+ },
+ "profile": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "The profile/configuration variant used for the request. Allows distinguishing different configuration variants and usage patterns.",
+ "maxLength": 255
+ },
+ "source": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "description": "The source/trigger of the request (e.g. 'metabot_agent' or 'document_generate_content' for Metabot). Indicates which API endpoint or workflow initiated the LLM call. Null for embeddings.",
+ "maxLength": 255
+ }
+ },
+ "required": [
+ "hashed_metabase_license_token",
+ "request_id",
+ "model_id",
+ "total_tokens",
+ "prompt_tokens",
+ "completion_tokens",
+ "estimated_costs_usd"
+ ],
+ "additionalProperties": false
+}
diff --git a/src/metabase/analytics/llm_token_usage.clj b/src/metabase/analytics/llm_token_usage.clj
index 69483f00d34..f538aba39b6 100644
--- a/src/metabase/analytics/llm_token_usage.clj
+++ b/src/metabase/analytics/llm_token_usage.clj
@@ -13,12 +13,14 @@
(def ^:private snowplow-arg-keys
[:request-id :model-id :total-tokens :prompt-tokens :completion-tokens :estimated-costs-usd
:cache-creation-tokens :cache-read-tokens :user-id :duration-ms :source :tag :session-id :profile
- :hashed-metabase-license-token])
+ :hashed-metabase-license-token :provider :model-name])
(def ^:private SnowplowArgs
[:map {:closed true}
[:request-id :string]
[:model-id :string]
+ [:provider :string]
+ [:model-name :string]
[:total-tokens ms/IntGreaterThanOrEqualToZero]
[:prompt-tokens ms/IntGreaterThanOrEqualToZero]
[:completion-tokens ms/IntGreaterThanOrEqualToZero]
@@ -35,7 +37,7 @@
(mu/defn track-snowplow!
"Track snowplow token_usage event."
- [{:keys [request-id model-id total-tokens prompt-tokens completion-tokens
+ [{:keys [request-id model-id provider model-name total-tokens prompt-tokens completion-tokens
cache-creation-tokens cache-read-tokens
estimated-costs-usd user-id duration-ms source tag session-id profile
hashed-metabase-license-token]}
@@ -45,6 +47,8 @@
(analytics.util/hashed-metabase-token-or-uuid))
:request-id request-id
:model-id model-id
+ :provider provider
+ :model-name model-name
:total-tokens total-tokens
:prompt-tokens prompt-tokens
:completion-tokens completion-tokens
diff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.clj
index b93274e5d8b..45aa989e514 100644
--- a/src/metabase/analytics/snowplow.clj
+++ b/src/metabase/analytics/snowplow.clj
@@ -51,7 +51,7 @@
:snowplow/action "1-0-0"
:snowplow/embed_share "1-0-2"
:snowplow/llm_usage "1-0-0"
- :snowplow/token_usage "1-0-5"
+ :snowplow/token_usage "1-0-6"
:snowplow/serialization "1-0-1"
:snowplow/simple_event "1-0-0"
:snowplow/cleanup "1-0-0"
@@ -146,6 +146,8 @@
[:map {:closed true}
[:request-id {:optional true} [:maybe :string]]
[:model-id {:optional true} [:maybe :string]]
+ [:provider {:optional true} [:maybe :string]]
+ [:model-name {:optional true} [:maybe :string]]
[:total-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]
[:prompt-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]
[:completion-tokens {:optional true} [:maybe ms/IntGreaterThanOrEqualToZero]]
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 8c9e56bdc3d..2a52d534c9e 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -222,20 +222,16 @@
Prometheus + Snowplow:
- `:model` - the model reference (e.g. `openrouter/anthropic/claude-haiku-4.5`)
- `:tag` - the specific purpose for which the tokens were used (e.g. 'agent', 'sql-fixing')
-
- Prometheus only:
- `:provider` - the provider type serving it (e.g. `openrouter`)
- - `:ai-proxy?` - whether the call went through the managed AI proxy
+ - `:ai-proxy?` - whether the call went through the managed AI proxy, reported as the `metabase` provider
Snowplow only:
+ - `:model-name` - the model as the provider names it (e.g. `anthropic/claude-haiku-4.5`)
- `:profile-id` - the profile id (e.g. `:internal`)
- `:request-id` - UUID string for this request
- `:session-id` - conversation UUID string
- `:source` - the source of the request (e.g., 'metabot_agent', 'document_generate_content').
- Indicates which API endpoint or workflow initiated the LLM call.
-
- Neither:
- - `:model-name` - the model as the provider names it (e.g. `anthropic/claude-haiku-4.5`)"
+ Indicates which API endpoint or workflow initiated the LLM call."
[{:keys [model model-name provider profile-id request-id session-id source tag ai-proxy?] :as tracking-opts}]
(let [start-ms (u/start-timer)]
(map (fn [part]
@@ -253,6 +249,7 @@
:profile (some-> profile-id name)
:model-id model
:provider (provider-label tracking-opts)
+ :model-name model-name
:prompt-tokens prompt
:completion-tokens completion
:cache-creation-tokens cache-creation
diff --git a/test/metabase/analytics/llm_token_usage_test.clj b/test/metabase/analytics/llm_token_usage_test.clj
index eb3be216705..d14f9b175ed 100644
--- a/test/metabase/analytics/llm_token_usage_test.clj
+++ b/test/metabase/analytics/llm_token_usage_test.clj
@@ -3,8 +3,12 @@
[clojure.test :refer :all]
[metabase.analytics-interface.core :as analytics]
[metabase.analytics.llm-token-usage :as llm-token-usage]
+ [metabase.analytics.snowplow :as snowplow]
[metabase.analytics.snowplow-test :as snowplow-test]
- [metabase.test :as mt]))
+ [metabase.test :as mt]
+ [metabase.util.json :as json])
+ (:import
+ (com.github.erosb.jsonsKema FormatValidationPolicy JsonParser SchemaLoader Validator ValidatorConfig)))
(set! *warn-on-reflection* true)
@@ -13,6 +17,8 @@
(def ^:private base-usage
{:request-id "abc123"
:model-id "anthropic/claude-haiku-4-5"
+ :provider "anthropic"
+ :model-name "claude-haiku-4-5"
:total-tokens 150
:prompt-tokens 100
:completion-tokens 50
@@ -45,6 +51,16 @@
(is (=? [{:data {"hashed_metabase_license_token" "my-custom-hash"}}]
(snowplow-test/pop-event-data-and-user-id!)))))))
+(defn- schema-violation
+ [event-data]
+ (let [path (str "snowplow/iglu-client-embedded/schemas/com.metabase/token_usage/jsonschema/"
+ (#'snowplow/schema->version :snowplow/token_usage))
+ ;; Otherwise the loader would fetch the meta-schema that `$schema` names over the network
+ schema (-> (json/decode (slurp path)) (dissoc "$schema") json/encode)]
+ (.validate (Validator/create (.load (SchemaLoader. schema))
+ (ValidatorConfig. FormatValidationPolicy/ALWAYS))
+ (.parse (JsonParser. (json/encode event-data))))))
+
(deftest track-snowplow!-all-fields-test
(testing "all Snowplow event fields are present and correct"
(mt/with-temporary-setting-values [premium-embedding-token nil
@@ -52,6 +68,8 @@
(snowplow-test/with-fake-snowplow-collector
(llm-token-usage/track-snowplow! {:request-id "deadbeef00"
:model-id "openai/gpt-4"
+ :provider "openai"
+ :model-name "gpt-4"
:total-tokens 300
:prompt-tokens 200
:completion-tokens 100
@@ -64,22 +82,38 @@
:tag "oss-sqlgen"
:session-id "session-abc"
:profile "internal"})
- (is (=? [{:user-id "42"
- :data {"hashed_metabase_license_token" "oss__uuid-for-test"
- "request_id" "deadbeef00"
- "model_id" "openai/gpt-4"
- "total_tokens" 300
- "prompt_tokens" 200
- "completion_tokens" 100
- "cache_creation_tokens" 250
- "cache_read_tokens" 900
- "estimated_costs_usd" 0.0
- "duration_ms" 1234
- "source" "oss_metabot"
- "tag" "oss-sqlgen"
- "session_id" "session-abc"
- "profile" "internal"}}]
- (snowplow-test/pop-event-data-and-user-id!)))))))
+ (let [events (snowplow-test/pop-event-data-and-user-id!)]
+ (is (=? [{:user-id "42"
+ :data {"hashed_metabase_license_token" "oss__uuid-for-test"
+ "request_id" "deadbeef00"
+ "model_id" "openai/gpt-4"
+ "provider" "openai"
+ "model_name" "gpt-4"
+ "total_tokens" 300
+ "prompt_tokens" 200
+ "completion_tokens" 100
+ "cache_creation_tokens" 250
+ "cache_read_tokens" 900
+ "estimated_costs_usd" 0.0
+ "duration_ms" 1234
+ "source" "oss_metabot"
+ "tag" "oss-sqlgen"
+ "session_id" "session-abc"
+ "profile" "internal"}}]
+ events))
+ (testing "and it validates against its schema"
+ (is (nil? (schema-violation (:data (first events)))))))))))
+
+(deftest track-snowplow!-matches-schema-test
+ (testing "an event without any optional field validates against its schema"
+ (snowplow-test/with-fake-snowplow-collector
+ (llm-token-usage/track-snowplow! base-usage)
+ (let [event (->> (snowplow-test/pop-event-data-and-user-id!)
+ (map :data)
+ (some #(when (contains? % "total_tokens") %)))]
+ (is (nil? (schema-violation event)))
+ (testing "and the schema rejects a field it does not declare"
+ (is (some? (schema-violation (assoc event "undeclared_field" "value")))))))))
;;; ------------------------------------------- track-prometheus! -------------------------------------------
@@ -161,6 +195,7 @@
:request-id "req-123"
:model-id "anthropic/claude-haiku-4-5"
:provider "anthropic"
+ :model-name "claude-haiku-4-5"
:tag "test-tag"
:prompt-tokens 100
:completion-tokens 50
@@ -169,6 +204,7 @@
(testing "Snowplow event fired"
(is (=? [{:data {"request_id" "req-123"
"model_id" "anthropic/claude-haiku-4-5"
+ "provider" "anthropic"
"total_tokens" 150
"prompt_tokens" 100}}]
(snowplow-test/pop-event-data-and-user-id!))))
@@ -203,6 +239,8 @@
:prometheus false
:request-id "req-456"
:model-id "openai/gpt-4"
+ :provider "openai"
+ :model-name "gpt-4"
:prompt-tokens 200
:completion-tokens 100
:total-tokens 300
@@ -212,7 +250,7 @@
(snowplow-test/pop-event-data-and-user-id!))))
(testing "no Prometheus metrics incremented"
(is (= 0.0 (mt/metric-value system :metabase-metabot/llm-input-tokens
- {:model "openai/gpt-4" :source "none" :provider "unknown"})))))))))
+ {:model "openai/gpt-4" :source "none" :provider "openai"})))))))))
(deftest track-token-usage!-both-false-error-test
(testing "throws when both :snowplow and :prometheus are false"
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 75ee671af88..a0116dcb0fa 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -1970,6 +1970,8 @@
tool-events (filter #(= "agent_used_tool" (get-in % [:data "event"])) events)]
(is (=? [{:user-id (str rasta-id)
:data {"model_id" "openrouter/test-model"
+ "provider" "openrouter"
+ "model_name" "test-model"
"total_tokens" 970
"prompt_tokens" 950
"completion_tokens" 20
@@ -2020,6 +2022,8 @@
token-events (filter #(contains? (:data %) "total_tokens") events)]
(is (=? [{:user-id (str rasta-id)
:data {"model_id" "openrouter/test-model"
+ "provider" "openrouter"
+ "model_name" "test-model"
"total_tokens" 60
"prompt_tokens" 50
"completion_tokens" 10
@@ -2028,12 +2032,14 @@
"source" "metabot_agent"
"tag" "test-tag"
"session_id" "00000000-0000-0000-0000-000000000002"}}]
- token-events))))))))))
-
-;;; ===================== Usage Log Tests =====================
+ token-events))
+ (testing "cache counts are 0 when the provider reports none"
+ (is (=? [{:data {"cache_creation_tokens" 0 "cache_read_tokens" 0}}]
+ token-events)))))))))))
-(deftest call-llm-usage-log-test
- (testing "call-llm and call-llm-structured log the provider type and the model as the provider names it"
+(deftest call-llm-provider-and-model-test
+ (testing (str "call-llm and call-llm-structured log the provider type and the model as the provider names it, "
+ "in ai_usage_log and on the token_usage event")
(llm.tu/with-connections [(assoc (llm.tu/connection "openrouter") :key "openrouter-1")
(llm.tu/connection "metabase")]
(let [model-ref "openrouter-1/anthropic/claude-sonnet-4.6"
@@ -2042,22 +2048,31 @@
{:type :tool-input :id "call-1" :function "json" :arguments {:answer "42"}}
{:type :usage :usage {:promptTokens 10 :completionTokens 5}}])
logged (atom [])]
- (mt/with-dynamic-fn-redefs [openrouter/openrouter (constantly response)
- self.claude/claude (constantly response)
- usage/log-ai-usage! #(swap! logged conj %)]
- (run! identity (self/call-llm model-ref nil [] {} {:tag "metabot_agent"}))
- (self/call-llm-structured model-ref [{:role "user" :content "test"}]
- {:type "object" :properties {:answer {:type "string"}}} 0.3 1024
- {:tag "metabot_agent"})
- (run! identity (self/call-llm "metabase/anthropic/claude-haiku-4-5" nil [] {} {:tag "metabot_agent"})))
- (is (=? (concat (repeat 2 {:model model-ref
- :provider "openrouter"
- :model-name "anthropic/claude-sonnet-4.6"})
- [{:model "metabase/anthropic/claude-haiku-4-5"
- :provider "anthropic"
- :model-name "claude-haiku-4-5"
- :ai-proxied true}])
- @logged))))))
+ (snowplow-test/with-fake-snowplow-collector
+ (mt/with-dynamic-fn-redefs [openrouter/openrouter (constantly response)
+ self.claude/claude (constantly response)
+ usage/log-ai-usage! #(swap! logged conj %)]
+ (run! identity (self/call-llm model-ref nil [] {} snowplow-tracking-opts))
+ (self/call-llm-structured model-ref [{:role "user" :content "test"}]
+ {:type "object" :properties {:answer {:type "string"}}} 0.3 1024
+ snowplow-tracking-opts)
+ (run! identity (self/call-llm "metabase/anthropic/claude-haiku-4-5" nil [] {} snowplow-tracking-opts)))
+ (is (=? (concat (repeat 2 {:model model-ref
+ :provider "openrouter"
+ :model-name "anthropic/claude-sonnet-4.6"})
+ [{:model "metabase/anthropic/claude-haiku-4-5"
+ :provider "anthropic"
+ :model-name "claude-haiku-4-5"
+ :ai-proxied true}])
+ @logged))
+ (is (=? (concat (repeat 2 {:data {"model_id" model-ref
+ "provider" "openrouter"
+ "model_name" "anthropic/claude-sonnet-4.6"}})
+ [{:data {"model_id" "metabase/anthropic/claude-haiku-4-5"
+ "provider" "metabase"
+ "model_name" "claude-haiku-4-5"}}])
+ (filter #(contains? (:data %) "total_tokens")
+ (snowplow-test/pop-event-data-and-user-id!)))))))))
;;; ----- gating: usage-limit + permission checks in call-llm-structured-with-trace -----
;;; (UXW-4126) The structured-with-trace path enforces usage limits unconditionally and