The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-1157 Update metabot prometheus and snowplow metrics to include both provider and model (#71259)
appleby74a9d5baebc1
diff --git a/src/metabase/metabot/self.clj b/src/metabase/metabot/self.cljindex 6042b9bffab..e13d52e7d3a 100644--- a/src/metabase/metabot/self.clj+++ b/src/metabase/metabot/self.clj@@ -129,7 +129,7 @@Prometheus + Snowplow:- `:profile-id` — the profile id (e.g. `:internal`)- - `:model` — the model (e.g. `anthropic/claude-haiku-4-5`)+ - `:model` — the model (e.g. `openrouter/anthropic/claude-haiku-4-5`)- `:tag` — the specific purpose for which the tokens were used (e.g. 'agent', 'sql-fixing')Snowplow only:@@ -142,7 +142,7 @@(map (fn [part](when (= (:type part) :usage)(let [usage (:usage part)- model (or (:model part) model "unknown")+ model (or model (:model part) "unknown")prompt (:promptTokens usage 0)completion (:completionTokens usage 0)](analytics/track-token-usage!@@ -245,7 +245,7 @@(let [{:keys [provider stream-fn model]} (parse-provider-model provider-and-model)](log/info "Calling LLM" {:provider provider :model model :parts (count parts) :tools (count tools):tool-choice tool-choice})- (let [tracking-opts (assoc tracking-opts :model model)+ (let [tracking-opts (assoc tracking-opts :model provider-and-model)streaming-opts (cond-> {:model model :input parts :tools (vals tools)}system-msg (assoc :system system-msg)(and (seq tools)@@ -282,13 +282,13 @@json-schema - JSON Schema map for the expected response shapetemperature - Sampling temperaturemax-tokens - Maximum tokens in the response- tracking-opts - See [[call-llm]] for fields; `:model` is added automatically+ tracking-opts - See [[report-token-usage-xf]] for fieldsReturns the parsed JSON map from the forced tool call."[provider-and-model messages json-schema temperature max-tokens tracking-opts](let [{:keys [provider stream-fn model]} (parse-provider-model provider-and-model)_ (log/info "Calling LLM (structured)" {:provider provider :model model :msg-count (count messages)})- tracking-opts (assoc tracking-opts :model model)+ tracking-opts (assoc tracking-opts :model provider-and-model)streaming-opts {:model model:input messages:schema json-schemadiff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.cljindex 2dad7d3b2c9..6a4a6a858ea 100644--- a/test/metabase/metabot/agent/core_test.clj+++ b/test/metabase/metabot/agent/core_test.clj@@ -485,7 +485,7 @@(is (== 0 (mt/metric-value system :metabase-metabot/agent-errors{:profile-id "internal"})))(is (== 2 (mt/metric-value system :metabase-metabot/llm-requests- {:model "anthropic/claude-haiku-4-5"+ {:model "openrouter/anthropic/claude-haiku-4-5":source "agent"})))(is (== 1 (:count (mt/metric-value system :metabase-metabot/agent-duration-ms{:profile-id "internal"}))))@@ -513,7 +513,7 @@(is (== 1 (mt/metric-value system :metabase-metabot/agent-errors{:profile-id "internal"})))(is (== 1 (mt/metric-value system :metabase-metabot/llm-requests- {:model "anthropic/claude-haiku-4-5"+ {:model "openrouter/anthropic/claude-haiku-4-5":source "agent"})))(is (== 1 (:count (mt/metric-value system :metabase-metabot/agent-duration-ms{:profile-id "internal"}))))@@ -638,7 +638,7 @@(let [events (snowplow-test/pop-event-data-and-user-id!)token-events (filter #(contains? (:data %) "total_tokens") events)](is (=? [{:user-id (str rasta-id)- :data {"model_id" "test-model"+ :data {"model_id" "openrouter/anthropic/claude-haiku-4-5""total_tokens" 120"prompt_tokens" 100"completion_tokens" 20@@ -649,7 +649,7 @@"tag" "agent""session_id" "00000000-0000-0000-0000-000000000001"}}{:user-id (str rasta-id)- :data {"model_id" "test-model"+ :data {"model_id" "openrouter/anthropic/claude-haiku-4-5""total_tokens" 180"prompt_tokens" 150"completion_tokens" 30diff --git a/test/metabase/metabot/api/document_test.clj b/test/metabase/metabot/api/document_test.cljindex 254e44b3d62..c067c3d1bd4 100644--- a/test/metabase/metabot/api/document_test.clj+++ b/test/metabase/metabot/api/document_test.clj@@ -20,7 +20,7 @@[{:type :start :id "msg-1"}{:type :text :text "No chart available"}{:type :usage :usage {:promptTokens 100 :completionTokens 20}- :model "test-model" :id "msg-1"}]))]+ :model "anthropic/claude-haiku-4-5" :id "msg-1"}]))](mt/user-http-request :crowberto:post 200 "metabot/document/native-generate-content"{:instructions "Show me sales data"}))@@ -29,11 +29,11 @@(is (== 1 (:sum (mt/metric-value system :metabase-metabot/agent-iterations{:profile-id "document-generate-content"}))))(is (== 1 (mt/metric-value system :metabase-metabot/llm-requests- {:model "anthropic/claude-haiku-4-5" :source "agent"})))+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))(is (== 100 (mt/metric-value system :metabase-metabot/llm-input-tokens- {:model "test-model" :source "agent"})))+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"})))(is (== 20 (mt/metric-value system :metabase-metabot/llm-output-tokens- {:model "test-model" :source "agent"}))))))+ {:model "openrouter/anthropic/claude-haiku-4-5" :source "agent"}))))))(deftest native-generate-content-snowplow-test(mt/with-premium-features #{:metabot-v3}@@ -44,13 +44,13 @@[{:type :start :id "msg-1"}{:type :text :text "No chart available"}{:type :usage :usage {:promptTokens 100 :completionTokens 20}- :model "test-model" :id "msg-1"}]))]+ :model "anthropic/claude-haiku-4-5" :id "msg-1"}]))](snowplow-test/with-fake-snowplow-collector(mt/user-http-request :rasta:post 200 "metabot/document/native-generate-content"{:instructions "Show me sales data"})(is (=? [{:user-id (str rasta-id)- :data {"model_id" "test-model"+ :data {"model_id" "openrouter/anthropic/claude-haiku-4-5""total_tokens" 120"prompt_tokens" 100"completion_tokens" 20diff --git a/test/metabase/metabot/example_question_generator_test.clj b/test/metabase/metabot/example_question_generator_test.cljindex 64dc2d52838..c63f0bde106 100644--- a/test/metabase/metabot/example_question_generator_test.clj+++ b/test/metabase/metabot/example_question_generator_test.clj@@ -148,7 +148,7 @@(deftest call-llm-prometheus-test(mt/with-prometheus-system! [_ system](mt/with-temporary-setting-values [llm-metabot-provider "openrouter/test-model"]- (let [labels {:model "test-model" :source "example-question-generation"}]+ (let [labels {:model "openrouter/test-model" :source "example-question-generation"}](testing "increments llm-requests and observes duration on success"(with-redefs [openrouter/openrouter(constantly (test-util/mock-llm-response@@ -176,7 +176,7 @@(snowplow-test/with-fake-snowplow-collector(#'native-generator/call-llm "test prompt")(is (=? [{:user-id (str rasta-id)- :data {"model_id" "test-model"+ :data {"model_id" "openrouter/test-model""total_tokens" 120"prompt_tokens" 100"completion_tokens" 20diff --git a/test/metabase/metabot/self_test.clj b/test/metabase/metabot/self_test.cljindex 6d4cbbed46b..c836fd64a3d 100644--- a/test/metabase/metabot/self_test.clj+++ b/test/metabase/metabot/self_test.clj@@ -559,7 +559,7 @@(deftest call-llm-prometheus-test(mt/with-prometheus-system! [_ system](with-redefs [self/retry-delay-ms (constantly 0)]- (let [labels {:model "test-model" :source "agent"}]+ (let [labels {:model "openrouter/test-model" :source "agent"}](testing "increments llm-requests and observes duration on success"(with-redefs [openrouter/openrouter (constantly (test-util/mock-llm-response [{:type :start :id "m1"}]))](run! identity (self/call-llm "openrouter/test-model" nil [] {} {:tag "agent"})))@@ -635,7 +635,7 @@(deftest call-llm-structured-prometheus-test(mt/with-prometheus-system! [_ system](with-redefs [self/retry-delay-ms (constantly 0)]- (let [labels {:model "test-model" :source "agent"}+ (let [labels {:model "openrouter/test-model" :source "agent"}success-mock (test-util/mock-llm-response[{:type :start :id "m1"}{:type :tool-input :id "call-1" :function "json"@@ -734,7 +734,7 @@(snowplow-test/with-fake-snowplow-collector(run! identity (self/call-llm "openrouter/test-model" nil [] test-util/TOOLS snowplow-tracking-opts))(is (=? [{:user-id (str rasta-id)- :data {"model_id" "test-model"+ :data {"model_id" "openrouter/test-model""total_tokens" 120"prompt_tokens" 100"completion_tokens" 20@@ -771,7 +771,7 @@1024snowplow-tracking-opts)(is (=? [{:user-id (str rasta-id)- :data {"model_id" "test-model"+ :data {"model_id" "openrouter/test-model""total_tokens" 60"prompt_tokens" 50"completion_tokens" 10