Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1157 Update metabot prometheus and snowplow metrics to include both provider and model (#71259)

appleby74a9d5baebc1
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.clj
index 6042b9bffab..e13d52e7d3a 100644
--- a/src/metabase/metabot/self.clj
+++ b/src/metabase/metabot/self.clj
@@ -129,7 +129,7 @@
Prometheus + Snowplow:
- `:profile-id` — the profile id (e.g. `:internal`)
- - `:model` — the model (e.g. `anthropic/claude-haiku-4-5`)
+ - `:model` — the model (e.g. `openrouter/anthropic/claude-haiku-4-5`)
- `:tag` — the specific purpose for which the tokens were used (e.g. 'agent', 'sql-fixing')
Snowplow only:
@@ -142,7 +142,7 @@
(map (fn [part]
(when (= (:type part) :usage)
(let [usage (:usage part)
- model (or (:model part) model "unknown")
+ model (or model (:model part) "unknown")
prompt (:promptTokens usage 0)
completion (:completionTokens usage 0)]
(analytics/track-token-usage!
@@ -245,7 +245,7 @@
(let [{:keys [provider stream-fn model]} (parse-provider-model provider-and-model)]
(log/info "Calling LLM" {:provider provider :model model :parts (count parts) :tools (count tools)
:tool-choice tool-choice})
- (let [tracking-opts (assoc tracking-opts :model model)
+ (let [tracking-opts (assoc tracking-opts :model provider-and-model)
streaming-opts (cond-> {:model model :input parts :tools (vals tools)}
system-msg (assoc :system system-msg)
(and (seq tools)
@@ -282,13 +282,13 @@
json-schema - JSON Schema map for the expected response shape
temperature - Sampling temperature
max-tokens - Maximum tokens in the response
- tracking-opts - See [[call-llm]] for fields; `:model` is added automatically
+ tracking-opts - See [[report-token-usage-xf]] for fields
Returns the parsed JSON map from the forced tool call."
[provider-and-model messages json-schema temperature max-tokens tracking-opts]
(let [{:keys [provider stream-fn model]} (parse-provider-model provider-and-model)
_ (log/info "Calling LLM (structured)" {:provider provider :model model :msg-count (count messages)})
- tracking-opts (assoc tracking-opts :model model)
+ tracking-opts (assoc tracking-opts :model provider-and-model)
streaming-opts {:model model
:input messages
:schema json-schema
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index 2dad7d3b2c9..6a4a6a858ea 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -485,7 +485,7 @@
(is (== 0 (mt/metric-value system :metabase-metabot/agent-errors
{:profile-id "internal"})))
(is (== 2 (mt/metric-value system :metabase-metabot/llm-requests
- {:model "anthropic/claude-haiku-4-5"
+ {:model "openrouter/anthropic/claude-haiku-4-5"
:source "agent"})))
(is (== 1 (:count (mt/metric-value system :metabase-metabot/agent-duration-ms
{:profile-id "internal"}))))
@@ -513,7 +513,7 @@
(is (== 1 (mt/metric-value system :metabase-metabot/agent-errors
{:profile-id "internal"})))
(is (== 1 (mt/metric-value system :metabase-metabot/llm-requests
- {:model "anthropic/claude-haiku-4-5"
+ {:model "openrouter/anthropic/claude-haiku-4-5"
:source "agent"})))
(is (== 1 (:count (mt/metric-value system :metabase-metabot/agent-duration-ms
{:profile-id "internal"}))))
@@ -638,7 +638,7 @@
(let [events (snowplow-test/pop-event-data-and-user-id!)
token-events (filter #(contains? (:data %) "total_tokens") events)]
(is (=? [{:user-id (str rasta-id)
- :data {"model_id" "test-model"
+ :data {"model_id" "openrouter/anthropic/claude-haiku-4-5"
"total_tokens" 120
"prompt_tokens" 100
"completion_tokens" 20
@@ -649,7 +649,7 @@
"tag" "agent"
"session_id" "00000000-0000-0000-0000-000000000001"}}
{:user-id (str rasta-id)
- :data {"model_id" "test-model"
+ :data {"model_id" "openrouter/anthropic/claude-haiku-4-5"
"total_tokens" 180
"prompt_tokens" 150
"completion_tokens" 30
diff --git a/test/metabase/metabot/api/document_test.clj b/test/metabase/metabot/api/document_test.clj
index 254e44b3d62..c067c3d1bd4 100644
--- a/test/metabase/metabot/api/document_test.clj
+++ b/test/metabase/metabot/api/document_test.clj
@@ -20,7 +20,7 @@
[{:type :start :id "msg-1"}
{:type :text :text "No chart available"}
{:type :usage :usage {:promptTokens 100 :completionTokens 20}
- :model "test-model" :id "msg-1"}]))]
+ :model "anthropic/claude-haiku-4-5" :id "msg-1"}]))]
(mt/user-http-request :crowberto
:post 200 "metabot/document/native-generate-content"
{:instructions "Show me sales data"}))
@@ -29,11 +29,11 @@
(is (== 1 (:sum (mt/metric-value system :metabase-metabot/agent-iterations
{:profile-id "document-generate-content"}))))
(is (== 1 (mt/metric-value system :metabase-metabot/llm-requests
- {:model "anthropic/claude-haiku-4-5" :source "agent"})))
+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))
(is (== 100 (mt/metric-value system :metabase-metabot/llm-input-tokens
- {:model "test-model" :source "agent"})))
+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))
(is (== 20 (mt/metric-value system :metabase-metabot/llm-output-tokens
- {:model "test-model" :source "agent"}))))))
+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"}))))))
(deftest native-generate-content-snowplow-test
(mt/with-premium-features #{:metabot-v3}
@@ -44,13 +44,13 @@
[{:type :start :id "msg-1"}
{:type :text :text "No chart available"}
{:type :usage :usage {:promptTokens 100 :completionTokens 20}
- :model "test-model" :id "msg-1"}]))]
+ :model "anthropic/claude-haiku-4-5" :id "msg-1"}]))]
(snowplow-test/with-fake-snowplow-collector
(mt/user-http-request :rasta
:post 200 "metabot/document/native-generate-content"
{:instructions "Show me sales data"})
(is (=? [{:user-id (str rasta-id)
- :data {"model_id" "test-model"
+ :data {"model_id" "openrouter/anthropic/claude-haiku-4-5"
"total_tokens" 120
"prompt_tokens" 100
"completion_tokens" 20
diff --git a/test/metabase/metabot/example_question_generator_test.clj b/test/metabase/metabot/example_question_generator_test.clj
index 64dc2d52838..c63f0bde106 100644
--- a/test/metabase/metabot/example_question_generator_test.clj
+++ b/test/metabase/metabot/example_question_generator_test.clj
@@ -148,7 +148,7 @@
(deftest call-llm-prometheus-test
(mt/with-prometheus-system! [_ system]
(mt/with-temporary-setting-values [llm-metabot-provider "openrouter/test-model"]
- (let [labels {:model "test-model" :source "example-question-generation"}]
+ (let [labels {:model "openrouter/test-model" :source "example-question-generation"}]
(testing "increments llm-requests and observes duration on success"
(with-redefs [openrouter/openrouter
(constantly (test-util/mock-llm-response
@@ -176,7 +176,7 @@
(snowplow-test/with-fake-snowplow-collector
(#'native-generator/call-llm "test prompt")
(is (=? [{:user-id (str rasta-id)
- :data {"model_id" "test-model"
+ :data {"model_id" "openrouter/test-model"
"total_tokens" 120
"prompt_tokens" 100
"completion_tokens" 20
diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.clj
index 6d4cbbed46b..c836fd64a3d 100644
--- a/test/metabase/metabot/self_test.clj
+++ b/test/metabase/metabot/self_test.clj
@@ -559,7 +559,7 @@
(deftest call-llm-prometheus-test
(mt/with-prometheus-system! [_ system]
(with-redefs [self/retry-delay-ms (constantly 0)]
- (let [labels {:model "test-model" :source "agent"}]
+ (let [labels {:model "openrouter/test-model" :source "agent"}]
(testing "increments llm-requests and observes duration on success"
(with-redefs [openrouter/openrouter (constantly (test-util/mock-llm-response [{:type :start :id "m1"}]))]
(run! identity (self/call-llm "openrouter/test-model" nil [] {} {:tag "agent"})))
@@ -635,7 +635,7 @@
(deftest call-llm-structured-prometheus-test
(mt/with-prometheus-system! [_ system]
(with-redefs [self/retry-delay-ms (constantly 0)]
- (let [labels {:model "test-model" :source "agent"}
+ (let [labels {:model "openrouter/test-model" :source "agent"}
success-mock (test-util/mock-llm-response
[{:type :start :id "m1"}
{:type :tool-input :id "call-1" :function "json"
@@ -734,7 +734,7 @@
(snowplow-test/with-fake-snowplow-collector
(run! identity (self/call-llm "openrouter/test-model" nil [] test-util/TOOLS snowplow-tracking-opts))
(is (=? [{:user-id (str rasta-id)
- :data {"model_id" "test-model"
+ :data {"model_id" "openrouter/test-model"
"total_tokens" 120
"prompt_tokens" 100
"completion_tokens" 20
@@ -771,7 +771,7 @@
1024
snowplow-tracking-opts)
(is (=? [{:user-id (str rasta-id)
- :data {"model_id" "test-model"
+ :data {"model_id" "openrouter/test-model"
"total_tokens" 60
"prompt_tokens" 50
"completion_tokens" 10