The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-1414 Ensure that edit_sql_query and replace_sql_query are included in :query_count (#73708)
appleby902ea41247f4
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj b/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.cljindex f2a6b5ee5d8..51dee510045 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj@@ -9,6 +9,7 @@[metabase-enterprise.metabot-analytics.queries :as analytics.queries][metabase.api.common :as api][metabase.metabot.persistence :as metabot-persistence]+ [metabase.metabot.tools :as metabot.tools][metabase.permissions.core :as perms][metabase.query-processor.parameters.dates :as qp.parameters.dates][metabase.slackbot.api :as slackbot.api]@@ -144,7 +145,7 @@(assoc row:search_count (analytics.queries/count-tool-invocations msgs "search"):query_count (analytics.queries/count-tool-invocations- msgs analytics.queries/new-query-tool-names))))+ msgs metabot.tools/query-generation-tool-names))))rows)))(defn list-conversations@@ -231,7 +232,7 @@:queries (analytics.queries/messages->generated-queries messages):search_count (analytics.queries/count-tool-invocations messages "search"):query_count (analytics.queries/count-tool-invocations- messages analytics.queries/new-query-tool-names)+ messages metabot.tools/query-generation-tool-names):ip_address (:ip_address conversation):embedding_hostname (:embedding_hostname conversation):embedding_path (:embedding_path conversation)diff --git a/enterprise/backend/src/metabase_enterprise/metabot_analytics/queries.clj b/enterprise/backend/src/metabase_enterprise/metabot_analytics/queries.cljindex 0a5c238d9c3..6db0acd0375 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_analytics/queries.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_analytics/queries.clj@@ -128,12 +128,6 @@[messages](into [] (mapcat message->generated-queries) messages))-(def new-query-tool-names- "Tools that construct a fresh query. Excludes `edit_sql_query` and- `replace_sql_query`, which refine an existing query rather than create- a new one."- #{"create_sql_query" "construct_notebook_query"})-(defn- tool-input-block? [block tool-names](and (= "tool-input" (:type block))(if (set? tool-names)diff --git a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.cljindex 21db5688a67..a78ca66a1be 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj@@ -523,9 +523,10 @@{:type "tool-output" :id call-id :result {:output "ok"}})(defn- with-query-count-fixture!- "Seed conversations that exercise `:query_count` (create_sql_query and- construct_notebook_query). Edit/replace tools and unrelated tools are- included to verify they are excluded from the count."+ "Seed conversations that exercise `:query_count` for every query-generation+ tool — create_sql_query, edit_sql_query, replace_sql_query, and+ construct_notebook_query. Non-query tool calls (search) are sprinkled in+ to verify they don't bump `:query_count`."[thunk](mt/with-premium-features #{:audit-app}(mt/with-temp [:model/User {test-user-id :id} {:email "[email protected]"@@ -545,11 +546,11 @@(insert-conversation! {:conversation-id convo-mixed:user-id test-user-id:created-at mar-2- :summary "one create + one notebook across messages"})+ :summary "create + edit + notebook across messages"})(insert-conversation! {:conversation-id convo-edits:user-id test-user-id:created-at mar-3- :summary "only edit/replace — should not count"})+ :summary "only edit/replace — counted as queries"});; convo-none: only a search, no new-query tools.(insert-message! {:conversation-id convo-none:created-at mar-1@@ -558,14 +559,16 @@:total-tokens 5:data [(search-input-block "call-s")(search-output-block "call-s")]})- ;; convo-mixed: one create_sql_query in msg 1, one construct_notebook_query- ;; in msg 2 alongside an excluded edit_sql_query.+ ;; convo-mixed: one search + create_sql_query in msg 1, then one edit_sql_query+ ;; and one construct_notebook_query in msg 2 — three counted queries.(insert-message! {:conversation-id convo-mixed:created-at mar-2:role "assistant":profile-id "internal":total-tokens 10- :data [(query-tool-input-block "call-a" "create_sql_query")+ :data [(search-input-block "call-mixed-s")+ (search-output-block "call-mixed-s")+ (query-tool-input-block "call-a" "create_sql_query")(query-tool-output-block "call-a")]})(insert-message! {:conversation-id convo-mixed:created-at mar-2@@ -576,13 +579,15 @@(query-tool-output-block "call-b")(query-tool-input-block "call-c" "construct_notebook_query")(query-tool-output-block "call-c")]})- ;; convo-edits: only edit + replace — both excluded from :query_count.+ ;; convo-edits: search + edit + replace(insert-message! {:conversation-id convo-edits:created-at mar-3:role "assistant":profile-id "internal":total-tokens 4- :data [(query-tool-input-block "call-e" "edit_sql_query")+ :data [(search-input-block "call-edits-s")+ (search-output-block "call-edits-s")+ (query-tool-input-block "call-e" "edit_sql_query")(query-tool-output-block "call-e")(query-tool-input-block "call-r" "replace_sql_query")(query-tool-output-block "call-r")]})@@ -596,14 +601,14 @@(deftest query-count-test(with-query-count-fixture!(fn [{:keys [test-user-id convo-none convo-mixed convo-edits]}]- (testing "list endpoint: counts create_sql_query + construct_notebook_query, excludes edit/replace"+ (testing "list endpoint: counts every query-generation tool (incl. edit/replace), excludes non-query tools like search"(let [response (mt/user-http-request :crowberto :get 200(format "ee/metabot-analytics/conversations?user_id=%s" test-user-id))by-id (into {} (map (juxt :conversation_id identity)) (:data response))]- (is (= {convo-none 0, convo-mixed 2, convo-edits 0}+ (is (= {convo-none 0, convo-mixed 3, convo-edits 2}(update-vals by-id :query_count)))))(testing "detail endpoint surfaces the same counts"- (doseq [[convo expected] [[convo-none 0] [convo-mixed 2] [convo-edits 0]]]+ (doseq [[convo expected] [[convo-none 0] [convo-mixed 3] [convo-edits 2]]](let [response (mt/user-http-request :crowberto :get 200(format "ee/metabot-analytics/conversations/%s" convo))](is (= expected (:query_count response))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_analytics/queries_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_analytics/queries_test.cljindex 8bea9a37f17..762a9dc392d 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_analytics/queries_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_analytics/queries_test.clj@@ -1,7 +1,8 @@(ns metabase-enterprise.metabot-analytics.queries-test(:require[clojure.test :refer [are deftest is testing]]- [metabase-enterprise.metabot-analytics.queries :as analytics.queries]))+ [metabase-enterprise.metabot-analytics.queries :as analytics.queries]+ [metabase.metabot.tools :as metabot.tools]))(set! *warn-on-reflection* true)@@ -260,7 +261,7 @@(testing "count-tool-invocations reaches tool names directly by :function"(is (= 1 (analytics.queries/count-tool-invocations [message] "search")))(is (= 1 (analytics.queries/count-tool-invocations- [message] analytics.queries/new-query-tool-names))))+ [message] metabot.tools/query-generation-tool-names))))(testing "messages->generated-queries surfaces the SQL tool call"(let [rows (analytics.queries/messages->generated-queries [message])](is (= 1 (count rows)))@@ -329,13 +330,11 @@:_type "TOOL_CALL":tool_calls [{:id "slack-1" :name "search"}]}]}]))(testing "accepts a set of tool names; a block counts if its :function is in the set"- ;; new-query-tool-names matches create_sql_query and construct_notebook_query,- ;; but not edit_sql_query / replace_sql_query.- (is (= 3 (analytics.queries/count-tool-invocations+ (is (= 5 (analytics.queries/count-tool-invocations[{:id 1 :data [{:type "tool-input" :function "create_sql_query" :id "a"}- {:type "tool-input" :function "edit_sql_query" :id "b"} ; excluded+ {:type "tool-input" :function "edit_sql_query" :id "b"}{:type "tool-input" :function "construct_notebook_query" :id "c"}]}- {:id 2 :data [{:type "tool-input" :function "replace_sql_query" :id "d"} ; excluded+ {:id 2 :data [{:type "tool-input" :function "replace_sql_query" :id "d"}{:type "tool-input" :function "create_sql_query" :id "e"}- {:type "tool-input" :function "search" :id "f"}]}] ; excluded- analytics.queries/new-query-tool-names)))))+ {:type "tool-input" :function "search" :id "f"}]}] ; non-query tool+ metabot.tools/query-generation-tool-names)))))