Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1414 Ensure that edit_sql_query and replace_sql_query are included in :query_count (#73708)

appleby902ea41247f4
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj b/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj
index f2a6b5ee5d8..51dee510045 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_analytics/conversations.clj
@@ -9,6 +9,7 @@
[metabase-enterprise.metabot-analytics.queries :as analytics.queries]
[metabase.api.common :as api]
[metabase.metabot.persistence :as metabot-persistence]
+ [metabase.metabot.tools :as metabot.tools]
[metabase.permissions.core :as perms]
[metabase.query-processor.parameters.dates :as qp.parameters.dates]
[metabase.slackbot.api :as slackbot.api]
@@ -144,7 +145,7 @@
(assoc row
:search_count (analytics.queries/count-tool-invocations msgs "search")
:query_count (analytics.queries/count-tool-invocations
- msgs analytics.queries/new-query-tool-names))))
+ msgs metabot.tools/query-generation-tool-names))))
rows)))
(defn list-conversations
@@ -231,7 +232,7 @@
:queries (analytics.queries/messages->generated-queries messages)
:search_count (analytics.queries/count-tool-invocations messages "search")
:query_count (analytics.queries/count-tool-invocations
- messages analytics.queries/new-query-tool-names)
+ messages metabot.tools/query-generation-tool-names)
:ip_address (:ip_address conversation)
:embedding_hostname (:embedding_hostname conversation)
:embedding_path (:embedding_path conversation)
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_analytics/queries.clj b/enterprise/backend/src/metabase_enterprise/metabot_analytics/queries.clj
index 0a5c238d9c3..6db0acd0375 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_analytics/queries.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_analytics/queries.clj
@@ -128,12 +128,6 @@
[messages]
(into [] (mapcat message->generated-queries) messages))
-(def new-query-tool-names
- "Tools that construct a fresh query. Excludes `edit_sql_query` and
- `replace_sql_query`, which refine an existing query rather than create
- a new one."
- #{"create_sql_query" "construct_notebook_query"})
-
(defn- tool-input-block? [block tool-names]
(and (= "tool-input" (:type block))
(if (set? tool-names)
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj
index 21db5688a67..a78ca66a1be 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_analytics/api_test.clj
@@ -523,9 +523,10 @@
{:type "tool-output" :id call-id :result {:output "ok"}})
(defn- with-query-count-fixture!
- "Seed conversations that exercise `:query_count` (create_sql_query and
- construct_notebook_query). Edit/replace tools and unrelated tools are
- included to verify they are excluded from the count."
+ "Seed conversations that exercise `:query_count` for every query-generation
+ tool — create_sql_query, edit_sql_query, replace_sql_query, and
+ construct_notebook_query. Non-query tool calls (search) are sprinkled in
+ to verify they don't bump `:query_count`."
[thunk]
(mt/with-premium-features #{:audit-app}
(mt/with-temp [:model/User {test-user-id :id} {:email "[email protected]"
@@ -545,11 +546,11 @@
(insert-conversation! {:conversation-id convo-mixed
:user-id test-user-id
:created-at mar-2
- :summary "one create + one notebook across messages"})
+ :summary "create + edit + notebook across messages"})
(insert-conversation! {:conversation-id convo-edits
:user-id test-user-id
:created-at mar-3
- :summary "only edit/replace — should not count"})
+ :summary "only edit/replace — counted as queries"})
;; convo-none: only a search, no new-query tools.
(insert-message! {:conversation-id convo-none
:created-at mar-1
@@ -558,14 +559,16 @@
:total-tokens 5
:data [(search-input-block "call-s")
(search-output-block "call-s")]})
- ;; convo-mixed: one create_sql_query in msg 1, one construct_notebook_query
- ;; in msg 2 alongside an excluded edit_sql_query.
+ ;; convo-mixed: one search + create_sql_query in msg 1, then one edit_sql_query
+ ;; and one construct_notebook_query in msg 2 — three counted queries.
(insert-message! {:conversation-id convo-mixed
:created-at mar-2
:role "assistant"
:profile-id "internal"
:total-tokens 10
- :data [(query-tool-input-block "call-a" "create_sql_query")
+ :data [(search-input-block "call-mixed-s")
+ (search-output-block "call-mixed-s")
+ (query-tool-input-block "call-a" "create_sql_query")
(query-tool-output-block "call-a")]})
(insert-message! {:conversation-id convo-mixed
:created-at mar-2
@@ -576,13 +579,15 @@
(query-tool-output-block "call-b")
(query-tool-input-block "call-c" "construct_notebook_query")
(query-tool-output-block "call-c")]})
- ;; convo-edits: only edit + replace — both excluded from :query_count.
+ ;; convo-edits: search + edit + replace
(insert-message! {:conversation-id convo-edits
:created-at mar-3
:role "assistant"
:profile-id "internal"
:total-tokens 4
- :data [(query-tool-input-block "call-e" "edit_sql_query")
+ :data [(search-input-block "call-edits-s")
+ (search-output-block "call-edits-s")
+ (query-tool-input-block "call-e" "edit_sql_query")
(query-tool-output-block "call-e")
(query-tool-input-block "call-r" "replace_sql_query")
(query-tool-output-block "call-r")]})
@@ -596,14 +601,14 @@
(deftest query-count-test
(with-query-count-fixture!
(fn [{:keys [test-user-id convo-none convo-mixed convo-edits]}]
- (testing "list endpoint: counts create_sql_query + construct_notebook_query, excludes edit/replace"
+ (testing "list endpoint: counts every query-generation tool (incl. edit/replace), excludes non-query tools like search"
(let [response (mt/user-http-request :crowberto :get 200
(format "ee/metabot-analytics/conversations?user_id=%s" test-user-id))
by-id (into {} (map (juxt :conversation_id identity)) (:data response))]
- (is (= {convo-none 0, convo-mixed 2, convo-edits 0}
+ (is (= {convo-none 0, convo-mixed 3, convo-edits 2}
(update-vals by-id :query_count)))))
(testing "detail endpoint surfaces the same counts"
- (doseq [[convo expected] [[convo-none 0] [convo-mixed 2] [convo-edits 0]]]
+ (doseq [[convo expected] [[convo-none 0] [convo-mixed 3] [convo-edits 2]]]
(let [response (mt/user-http-request :crowberto :get 200
(format "ee/metabot-analytics/conversations/%s" convo))]
(is (= expected (:query_count response))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_analytics/queries_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_analytics/queries_test.clj
index 8bea9a37f17..762a9dc392d 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_analytics/queries_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_analytics/queries_test.clj
@@ -1,7 +1,8 @@
(ns metabase-enterprise.metabot-analytics.queries-test
(:require
[clojure.test :refer [are deftest is testing]]
- [metabase-enterprise.metabot-analytics.queries :as analytics.queries]))
+ [metabase-enterprise.metabot-analytics.queries :as analytics.queries]
+ [metabase.metabot.tools :as metabot.tools]))
(set! *warn-on-reflection* true)
@@ -260,7 +261,7 @@
(testing "count-tool-invocations reaches tool names directly by :function"
(is (= 1 (analytics.queries/count-tool-invocations [message] "search")))
(is (= 1 (analytics.queries/count-tool-invocations
- [message] analytics.queries/new-query-tool-names))))
+ [message] metabot.tools/query-generation-tool-names))))
(testing "messages->generated-queries surfaces the SQL tool call"
(let [rows (analytics.queries/messages->generated-queries [message])]
(is (= 1 (count rows)))
@@ -329,13 +330,11 @@
:_type "TOOL_CALL"
:tool_calls [{:id "slack-1" :name "search"}]}]}]))
(testing "accepts a set of tool names; a block counts if its :function is in the set"
- ;; new-query-tool-names matches create_sql_query and construct_notebook_query,
- ;; but not edit_sql_query / replace_sql_query.
- (is (= 3 (analytics.queries/count-tool-invocations
+ (is (= 5 (analytics.queries/count-tool-invocations
[{:id 1 :data [{:type "tool-input" :function "create_sql_query" :id "a"}
- {:type "tool-input" :function "edit_sql_query" :id "b"} ; excluded
+ {:type "tool-input" :function "edit_sql_query" :id "b"}
{:type "tool-input" :function "construct_notebook_query" :id "c"}]}
- {:id 2 :data [{:type "tool-input" :function "replace_sql_query" :id "d"} ; excluded
+ {:id 2 :data [{:type "tool-input" :function "replace_sql_query" :id "d"}
{:type "tool-input" :function "create_sql_query" :id "e"}
- {:type "tool-input" :function "search" :id "f"}]}] ; excluded
- analytics.queries/new-query-tool-names)))))
+ {:type "tool-input" :function "search" :id "f"}]}] ; non-query tool
+ metabot.tools/query-generation-tool-names)))))