The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
OSS sqlgen snowplow test cleanups (#68929)
appleby222b6b1bd00f
diff --git a/src/metabase/llm/costs.clj b/src/metabase/llm/costs.cljindex 406ecbe1cb8..c8577c0bdba 100644--- a/src/metabase/llm/costs.clj+++ b/src/metabase/llm/costs.clj@@ -5,14 +5,38 @@(def ^:private model-pricing"Model pricing by model family. Prices are in USD. :input and :output are per :scale tokens."- {"anthropic/claude-opus-4-5" {:input 5.00 :output 25.00 :scale 1000000}- "anthropic/claude-opus-4-1" {:input 15.00 :output 75.00 :scale 1000000}- "anthropic/claude-opus-4" {:input 15.00 :output 75.00 :scale 1000000}- "anthropic/claude-sonnet-4-5" {:input 3.00 :output 15.00 :scale 1000000}- "anthropic/claude-sonnet-4" {:input 3.00 :output 15.00 :scale 1000000}- "anthropic/claude-haiku-4-5" {:input 1.00 :output 5.00 :scale 1000000}- "anthropic/claude-haiku-3-5" {:input 0.80 :output 4.00 :scale 1000000}- "anthropic/claude-haiku-3" {:input 0.25 :output 1.25 :scale 1000000}})+ {"anthropic/claude-3-5-haiku" {:input 0.80, :output 4.00, :scale 1000000},+ "anthropic/claude-3-7-sonnet" {:input 3.00, :output 15.00, :scale 1000000},+ "anthropic/claude-3-haiku" {:input 0.25, :output 1.25, :scale 1000000},+ "anthropic/claude-haiku-3" {:input 0.25, :output 1.25, :scale 1000000},+ "anthropic/claude-haiku-3-5" {:input 0.80, :output 4.00, :scale 1000000},+ "anthropic/claude-haiku-4-5" {:input 1.00, :output 5.00, :scale 1000000},+ "anthropic/claude-opus-4" {:input 15.00, :output 75.00, :scale 1000000},+ "anthropic/claude-opus-4-1" {:input 15.00, :output 75.00, :scale 1000000},+ "anthropic/claude-opus-4-5" {:input 5.00, :output 25.00, :scale 1000000},+ "anthropic/claude-sonnet-4" {:input 3.00, :output 15.00, :scale 1000000},+ "anthropic/claude-sonnet-4-5" {:input 3.00, :output 15.00, :scale 1000000}})++(comment+ ;; Eval the following forms to get an updated model-pricing table with keys for new anthropic models. Unfortunately,+ ;; model prices don't seem to be fetchable via an API at time of writing, so you have have to look them up for any+ ;; new models at the following URL (we could scrape it, but seems fragile):+ ;;+ ;; https://platform.claude.com/docs/en/about-claude/pricing++ ;; First configure metabase.llm.settings/llm-anthropic-api-key with a valid key+ ((requiring-resolve 'metabase.llm.settings/llm-anthropic-api-key!) "<api-key>")++ (defn- merge-pricing [new-models]+ (merge new-models model-pricing))++ (->> ((requiring-resolve 'metabase.llm.anthropic/list-models))+ :models+ (map (fn [{:keys [id]}]+ [(normalize-model-name id) {:input nil :output nil :scale 1000000}]))+ (into {})+ merge-pricing+ (into (sorted-map))))(defn- normalize-model-name"Normalize a raw model name to the `provider/family` format used in the pricing table.diff --git a/test/metabase/llm/anthropic_test.clj b/test/metabase/llm/anthropic_test.cljindex 29fe5f13245..b3f095851eb 100644--- a/test/metabase/llm/anthropic_test.clj+++ b/test/metabase/llm/anthropic_test.clj@@ -112,6 +112,30 @@#"not configured"(anthropic/chat-completion {:messages [{:role "user" :content "test"}]}))))))+(deftest chat-completion-returns-usage-test+ (testing "chat-completion returns result, usage, and duration"+ (let [mock-response {:body {:id "msg_123"+ :model "claude-sonnet-4-5-20250929"+ :content [{:type "tool_use"+ :id "tool_123"+ :name "generate_sql"+ :input {:sql "SELECT * FROM users"+ :explanation "Fetches all users"}}]+ :usage {:input_tokens 1500+ :output_tokens 250}}}]+ (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test-key"+ llm-anthropic-model "claude-sonnet-4-5-20250929"]+ (with-redefs [http/post (constantly mock-response)]+ (let [result (anthropic/chat-completion {:system "You are a SQL expert"+ :messages [{:role "user" :content "get all users"}]})]+ (is (=? {:result {:sql "SELECT * FROM users"+ :explanation "Fetches all users"}+ :duration-ms #(and (number? %) (pos? %))+ :usage {:model "claude-sonnet-4-5-20250929"+ :prompt 1500+ :completion 250}}+ result))))))))+;;; ------------------------------------------- list-models Tests -------------------------------------------(deftest list-models-requires-api-key-test@@ -148,27 +172,3 @@{:id "claude-sonnet-4-5-20250929" :display_name "Claude Sonnet 4.5"}{:id "claude-sonnet-4-20250514" :display_name "Claude Sonnet 4"}](:models result)))))))))--(deftest chat-completion-returns-usage-test- (testing "chat-completion returns result, usage, and duration"- (let [mock-response {:body {:id "msg_123"- :model "claude-sonnet-4-5-20250929"- :content [{:type "tool_use"- :id "tool_123"- :name "generate_sql"- :input {:sql "SELECT * FROM users"- :explanation "Fetches all users"}}]- :usage {:input_tokens 1500- :output_tokens 250}}}]- (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test-key"- llm-anthropic-model "claude-sonnet-4-5-20250929"]- (with-redefs [http/post (constantly mock-response)]- (let [result (anthropic/chat-completion {:system "You are a SQL expert"- :messages [{:role "user" :content "get all users"}]})]- (is (=? {:result {:sql "SELECT * FROM users"- :explanation "Fetches all users"}- :duration-ms #(and (number? %) (pos? %))- :usage {:model "claude-sonnet-4-5-20250929"- :prompt 1500- :completion 250}}- result))))))))diff --git a/test/metabase/llm/api_test.clj b/test/metabase/llm/api_test.cljindex 912b0c4d0af..a45c8818710 100644--- a/test/metabase/llm/api_test.clj+++ b/test/metabase/llm/api_test.clj@@ -3,12 +3,10 @@[clojure.set :as set][clojure.string :as str][clojure.test :refer :all]- [metabase.analytics.core :as analytics]- [metabase.analytics.snowplow :as snowplow]+ [metabase.analytics.snowplow-test :as snowplow-test][metabase.llm.anthropic :as llm.anthropic][metabase.llm.api :as api][metabase.llm.context :as llm.context]- [metabase.premium-features.core :as premium-features][metabase.test :as mt]))(set! *warn-on-reflection* true)@@ -148,47 +146,53 @@;;; ------------------------------------------- Snowplow Tests -------------------------------------------+(defn- token-usage-event? [event]+ (-> event+ :data+ (contains? "total_tokens")))++(defn- simple-event? [event]+ (-> event+ :data+ (contains? "event")))+(deftest generate-sql-snowplow-success-test(testing "successful /generate-sql call tracks both token_usage and simple_event"(mt/with-temp [:model/Database db {:engine :postgres}:model/Table table {:db_id (:id db) :name "users" :schema "public"}:model/Field _ {:table_id (:id table) :name "id" :base_type :type/Integer}:model/Field _ {:table_id (:id table) :name "name" :base_type :type/Text}]- (let [tracked-events (atom [])- mock-chat-response {:result {:sql "SELECT * FROM users"}- :usage {:model "anthropic/claude-sonnet-4-5"- :prompt 1000- :completion 200}+ (let [mock-chat-response {:result {:sql "SELECT * FROM users"}+ :usage {:model "claude-sonnet-4-5-20250929"+ :prompt 1000+ :completion 200}:duration-ms 500}](mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test"]- (with-redefs [llm.anthropic/chat-completion (constantly mock-chat-response)- snowplow/track-event! (fn [schema data user-id]- (swap! tracked-events conj {:schema schema- :data data- :user-id user-id}))]- (let [response (mt/user-http-request :rasta :post 200 "llm/generate-sql"- {:prompt "get all users"- :database_id (:id db)- :referenced_entities [{:model "table" :id (:id table)}]})]- (is (= "SELECT * FROM users" (:sql response)))- (testing "token_usage event"- (let [token-events (filter #(= :snowplow/token_usage (:schema %)) @tracked-events)]- (is (=? [{:schema :snowplow/token_usage- :data {:model-id "anthropic/claude-sonnet-4-5"- :prompt-tokens 1000- :completion-tokens 200- :total-tokens 1200- :duration-ms 500- :source "oss_metabot"- :tag "oss-sqlgen"}}]- token-events))))- (testing "simple_event"- (let [simple-events (filter #(= :snowplow/simple_event (:schema %)) @tracked-events)]- (is (=? [{:schema :snowplow/simple_event- :data {:event "metabot_oss_sqlgen_used"- :duration_ms int?- :result "success"- :event_detail "postgres"}}]+ (snowplow-test/with-fake-snowplow-collector+ (with-redefs [llm.anthropic/chat-completion (constantly mock-chat-response)]+ (let [response (mt/user-http-request :rasta :post 200 "llm/generate-sql"+ {:prompt "get all users"+ :database_id (:id db)+ :referenced_entities [{:model "table" :id (:id table)}]})+ events (snowplow-test/pop-event-data-and-user-id!)+ token-events (filter token-usage-event? events)+ simple-events (filter simple-event? events)]+ (is (= "SELECT * FROM users" (:sql response)))+ (testing "token_usage event"+ (is (=? [{:data {"model_id" "claude-sonnet-4-5-20250929"+ "prompt_tokens" 1000+ "completion_tokens" 200+ "total_tokens" 1200+ "estimated_costs_usd" #(and (number? %) (pos? %))+ "duration_ms" 500+ "source" "oss_metabot"+ "tag" "oss-sqlgen"}}]+ token-events)))+ (testing "simple_event"+ (is (=? [{:data {"event" "metabot_oss_sqlgen_used"+ "duration_ms" int?+ "result" "success"+ "event_detail" "postgres"}}]simple-events)))))))))))(deftest generate-sql-snowplow-failure-test@@ -196,80 +200,67 @@(mt/with-temp [:model/Database db {:engine :postgres}:model/Table table {:db_id (:id db) :name "users" :schema "public"}:model/Field _ {:table_id (:id table) :name "id" :base_type :type/Integer}]- (let [tracked-events (atom [])]- (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test"]- (with-redefs [llm.anthropic/chat-completion (fn [_] (throw (Exception. "API error")))- snowplow/track-event! (fn [schema data user-id]- (swap! tracked-events conj {:schema schema- :data data- :user-id user-id}))]+ (mt/with-temporary-setting-values [llm-anthropic-api-key "sk-ant-test"]+ (snowplow-test/with-fake-snowplow-collector+ (with-redefs [llm.anthropic/chat-completion (fn [_] (throw (Exception. "API error")))](mt/user-http-request :rasta :post 500 "llm/generate-sql"- {:prompt "get all users"- :database_id (:id db)+ {:prompt "get all users"+ :database_id (:id db):referenced_entities [{:model "table" :id (:id table)}]})- (testing "no token_usage event on failure (chat-completion call failed)"- (let [token-events (filter #(= :snowplow/token_usage (:schema %)) @tracked-events)]- (is (empty? token-events))))- (testing "simple_event with failure result"- (let [simple-events (filter #(= :snowplow/simple_event (:schema %)) @tracked-events)]- (is (=? [{:schema :snowplow/simple_event- :data {:event "metabot_oss_sqlgen_used"- :duration_ms int?- :result "failure"- :event_detail "postgres"}}]+ (let [events (snowplow-test/pop-event-data-and-user-id!)+ token-events (filter token-usage-event? events)+ simple-events (filter simple-event? events)]+ (testing "no token_usage event on failure (chat-completion call failed)"+ (is (empty? token-events)))+ (testing "simple_event with failure result"+ (is (=? [{:data {"event" "metabot_oss_sqlgen_used"+ "duration_ms" int?+ "result" "failure"+ "event_detail" "postgres"}}]simple-events))))))))));;; ------------------------------------------- Token Usage Tracking Tests -------------------------------------------(deftest track-token-usage-with-uuid-test(testing "tracks usage with analytics uuid when no premium token is available"- (let [tracked-events (atom [])- test-analytics-uuid "test-analytics-uuid-12345"]- (with-redefs [snowplow/track-event! (fn [schema data user-id]- (swap! tracked-events conj {:schema schema- :data data- :user-id user-id}))- premium-features/premium-embedding-token (constantly nil)- analytics/analytics-uuid (constantly test-analytics-uuid)]- (#'api/track-token-usage! {:model "anthropic/claude-sonnet-4-5"- :prompt 1000- :completion 500- :duration-ms 1234- :user-id 42- :source "oss_metabot"- :tag "oss-sqlgen"})- (is (= 1 (count @tracked-events)))- (let [{:keys [schema data user-id]} (first @tracked-events)]- (is (= :snowplow/token_usage schema))- (is (= 42 user-id))- (is (=? {:hashed-metabase-license-token (str "oss__" test-analytics-uuid)- :request-id #"[a-h0-9]{32}" ; UUID hex format (no dashes)- :model-id "anthropic/claude-sonnet-4-5"- :total-tokens 1500- :prompt-tokens 1000- :completion-tokens 500- :estimated-costs-usd pos?- :duration-ms 1234- :source "oss_metabot"- :tag "oss-sqlgen"}- data)))))))+ (let [test-analytics-uuid "test-analytics-uuid-12345"]+ (mt/with-temporary-setting-values [premium-embedding-token nil+ analytics-uuid test-analytics-uuid]+ (snowplow-test/with-fake-snowplow-collector+ (#'api/track-token-usage! {:model "claude-sonnet-4-5-20250929"+ :prompt 1000+ :completion 500+ :duration-ms 1234+ :user-id 42+ :source "oss_metabot"+ :tag "oss-sqlgen"})+ (is (=? [{:user-id "42"+ :data {"hashed_metabase_license_token" (str "oss__" test-analytics-uuid)+ "request_id" #"[a-h0-9]{32}" ; UUID hex format (no dashes)+ "model_id" "claude-sonnet-4-5-20250929"+ "total_tokens" 1500+ "prompt_tokens" 1000+ "completion_tokens" 500+ "estimated_costs_usd" #(and (number? %) (pos? %))+ "duration_ms" 1234+ "source" "oss_metabot"+ "tag" "oss-sqlgen"}}]+ (->> (snowplow-test/pop-event-data-and-user-id!)+ (filter token-usage-event?)))))))))(deftest track-token-usage-with-premium-token-test(testing "hashes premium token when available"- (let [tracked-events (atom [])]- (with-redefs [snowplow/track-event! (fn [schema data user-id]- (swap! tracked-events conj {:schema schema- :data data- :user-id user-id}))- premium-features/premium-embedding-token (constantly "test-premium-token")]- (#'api/track-token-usage! {:model "anthropic/claude-sonnet-4-5"- :prompt 100- :completion 50- :duration-ms 100- :user-id 1- :source "test"- :tag "test"})- (let [{:keys [data]} (first @tracked-events)]+ (mt/with-random-premium-token! [premium-token]+ (mt/with-temporary-setting-values [premium-embedding-token premium-token]+ (snowplow-test/with-fake-snowplow-collector+ (#'api/track-token-usage! {:model "claude-sonnet-4-5-20250929"+ :prompt 100+ :completion 50+ :duration-ms 100+ :user-id 1+ :source "test"+ :tag "test"});; Should be a SHA-256 hash (64 hex chars), not "oss__*"- (is (=? {:hashed-metabase-license-token #"[0-9a-f]{64}"}- data)))))))+ (is (=? [{:data {"hashed_metabase_license_token" #"[0-9a-f]{64}"}}]+ (->> (snowplow-test/pop-event-data-and-user-id!)+ (filter token-usage-event?)))))))))diff --git a/test/metabase/llm/costs_test.clj b/test/metabase/llm/costs_test.cljindex a5b124a91b3..732f6ebb2f4 100644--- a/test/metabase/llm/costs_test.clj+++ b/test/metabase/llm/costs_test.clj@@ -6,7 +6,7 @@(set! *warn-on-reflection* true)-(deftest ^:parallel estimate-sanity-test+(deftest ^:parallel estimate-simplified-models-test(testing "known models calculate correct costs"(is (=? (=?/approx [0.030 0.0001])(costs/estimate {:model "anthropic/claude-opus-4-5"@@ -36,7 +36,7 @@(deftest ^:parallel estimate-zero-tokens-test(testing "zero tokens returns zero cost"(is (= 0.0- (costs/estimate {:model "anthropic/claude-sonnet-4-5"+ (costs/estimate {:model "claude-sonnet-4-5-20250929":prompt 0:completion 0})))))