Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Check queries before Metabot reports a chart as ready

Andrei Simionescu7e35c0c073e3
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj
index 0ae23a61295..ba5e69205ec 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj
@@ -3,18 +3,25 @@
[clojure.string :as str]
[clojure.test :refer [deftest is testing]]
[metabase-enterprise.sandbox.test-util :as sandbox.tu]
+ [metabase-enterprise.test :as met]
[metabase.api.common :as api]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.metabot.metadata-perms :as metabot.perms]
+ [metabase.metabot.test-util :as test-util]
[metabase.metabot.tools.construct :as construct]
+ [metabase.metabot.tools.slackbot-query :as slackbot-query]
[metabase.permissions.core :as perms]
[metabase.query-processor :as qp]
+ [metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.test :as mt]
[metabase.util :as u]
+ [metabase.util.log.capture :as log.capture]
[toucan2.core :as t2]))
+(set! *warn-on-reflection* true)
+
(defn- sandbox-query [table fields]
(let [mp (mt/metadata-provider)]
(lib/with-fields (lib/query mp (lib.metadata/table mp (mt/id table)))
@@ -106,3 +113,79 @@
(is (= :completed (:status actual-result)))
(is (seq (mt/rows actual-result)))
(is (= (set (mt/rows control-result)) (set (mt/rows actual-result)))))))))
+
+(defn- count-query [table]
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql"
+ :source-table ["Metabot sandbox" "PUBLIC" table]
+ :aggregation [["count" {}]]}]})
+
+(defn- orders-sandbox-query []
+ (let [mp (mt/metadata-provider)]
+ (lib/filter (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ (lib/> (lib.metadata/field mp (mt/id :orders :total)) 10))))
+
+(defn- check-sandbox-attribute-error! [attributes message]
+ (met/with-gtaps! {:gtaps {:venues {:query (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/table mp (mt/id :venues))))
+ :remappings {:cat ["variable" [:field (mt/id :venues :category_id) nil]]}}}
+ :attributes attributes}
+ (t2/update! :model/Database (mt/id) {:name "Metabot sandbox"})
+ (let [query (count-query "VENUES")]
+ (doseq [tool [construct/construct-notebook-query-tool
+ slackbot-query/slackbot-construct-notebook-query-tool]]
+ (let [built (get-in (test-util/call-tool tool {:query query :reasoning "count venues"})
+ [:structured-output :query])]
+ (is (map? built))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo message (qp/process-query built))))))))
+
+(deftest construct-tools-allow-missing-sandbox-attribute-test
+ (check-sandbox-attribute-error! {"something_random" 50} #"Query requires user attribute `cat`"))
+
+(deftest construct-tools-preprocess-sandboxed-queries-test
+ (met/with-gtaps! {:gtaps {:orders {:query (orders-sandbox-query)}}}
+ (t2/update! :model/Database (mt/id) {:name "Metabot sandbox"})
+ (let [compilations (atom 0)
+ executions (atom 0)
+ query (count-query "ORDERS")]
+ (mt/with-dynamic-fn-redefs [qp.compile/compile-preprocessed (fn [& _]
+ (swap! compilations inc)
+ (throw (ex-info "Unexpected SQL compilation" {})))]
+ (binding [qp.pipeline/*execute* (fn [& _]
+ (swap! executions inc)
+ (throw (ex-info "Unexpected query execution" {})))]
+ (log.capture/with-log-messages-for-level [logs [metabase.metabot.tools.construct :warn]]
+ (doseq [tool [construct/construct-notebook-query-tool
+ slackbot-query/slackbot-construct-notebook-query-tool]]
+ (let [built (get-in (test-util/call-tool tool {:query query :reasoning "count orders"})
+ [:structured-output :query])]
+ (is (= 1 (lib/stage-count built)))
+ (is (= (mt/id :orders) (get-in built [:stages 0 :source-table])))))
+ (is (empty? (logs)))))
+ (is (zero? @compilations))
+ (is (zero? @executions))))))
+
+(deftest construct-tools-allow-invalid-sandbox-attribute-test
+ (check-sandbox-attribute-error! {"cat" "a"} #"User attribute `cat` value `a` cannot be coerced"))
+
+(deftest construct-tools-load-native-sandbox-metadata-test
+ (doseq [tool [construct/construct-notebook-query-tool
+ slackbot-query/slackbot-construct-notebook-query-tool]]
+ (met/with-gtaps! {:gtaps {:venues {:query (lib/native-query
+ (mt/metadata-provider)
+ "SELECT ID, NAME, CATEGORY_ID, PRICE FROM VENUES WHERE PRICE = 1")}}}
+ (t2/update! :model/Database (mt/id) {:name "Metabot sandbox"})
+ (let [card-id (t2/select-one-fn :card_id :model/Sandbox :table_id (mt/id :venues))
+ executions (atom 0)
+ execute qp.pipeline/*execute*
+ query (count-query "VENUES")]
+ (is (nil? (t2/select-one-fn :result_metadata :model/Card :id card-id)))
+ (binding [qp.pipeline/*execute* (fn [driver query respond]
+ (swap! executions inc)
+ (execute driver query respond))]
+ (doseq [expected-executions [1 0]]
+ (reset! executions 0)
+ (is (map? (get-in (test-util/call-tool tool {:query query :reasoning "count venues"})
+ [:structured-output :query])))
+ (is (= expected-executions @executions))
+ (is (seq (t2/select-one-fn :result_metadata :model/Card :id card-id)))))))))
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index a0ec40341f9..566291c7359 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -36,9 +36,11 @@
[metabase.query-processor.core :as qp]
[metabase.util :as u]
[metabase.util.json :as json]
+ [metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
- [metabase.util.match :as match])
+ [metabase.util.match :as match]
+ [metabase.util.quoting :as quoting])
(:import
(java.sql SQLException)))
@@ -1098,18 +1100,62 @@
(ex-info (str (ex-message e) "\n" hint) (ex-data e) e)
e))
+(def ^:private preprocessing-message-limit 300)
+
+(defn- preprocessing-rejection-message
+ [cause]
+ (if (contains? #{:invalid-query :unsupported-feature} (:type (ex-data cause)))
+ (quoting/quoted (u/truncate (or (ex-message cause) "Invalid query.") preprocessing-message-limit))
+ (when (instance? clojure.lang.ExceptionInfo cause)
+ (case (ex-message cause)
+ "Incompatible metric"
+ (str "A metric is incompatible with this query stage. Use a metric that applies to the stage's "
+ "source table and columns.")
+
+ "Failed to replace metric"
+ (str "Metabase could not expand a metric reference. Check that it is in an aggregation and applies "
+ "to this stage's source.")
+
+ nil))))
+
+(defn- check-query-preprocessing!
+ [query]
+ (try
+ (qp/preprocess query)
+ (catch Throwable e
+ (let [chain (u/full-exception-chain e)
+ message (preprocessing-rejection-message (ex-cause e))]
+ (if (and (= :qp (:type (ex-data e)))
+ (:fn (ex-data e))
+ message
+ (not-any? #(or (= 403 (:status-code (ex-data %)))
+ (instance? java.sql.SQLException %)
+ (contains? (ex-data %) :toucan2/context-trace)) chain))
+ (throw (ex-info (str "Metabase could not prepare this query to run: "
+ message)
+ {:agent-error? true
+ :error :query-preprocessing-failed
+ :status-code 400}
+ e))
+ (log/warn e "Could not preprocess the constructed Metabot query"))))))
+
(defn execute-representations-query
"Run `external-query` through the representations pipeline. See
[[execute-representations-query*]] for the pipeline itself.
`opts` may carry `:recovery-hint`, a function from an agent error's `ex-data` to a sentence
string (or nil) telling the caller's agent how to recover; the sentence is appended to the
- agent error's message. Omitting it leaves agent errors with the pipeline's bare statement."
+ agent error's message. Omitting it leaves agent errors with the pipeline's bare statement.
+
+ `:preprocess-check?` checks the built query before returning it. Known query errors become
+ agent errors; other preprocessing failures are logged and ignored."
([external-query]
(execute-representations-query external-query nil))
- ([external-query {:keys [recovery-hint]}]
+ ([external-query {:keys [recovery-hint preprocess-check?]}]
(try
- (execute-representations-query* external-query)
+ (u/prog1 (execute-representations-query* external-query)
+ (when preprocess-check?
+ (check-query-preprocessing! (get-in <> [:structured-output :query]))))
(catch clojure.lang.ExceptionInfo e
(throw (if (:agent-error? (ex-data e))
(with-recovery-hint e recovery-hint)
@@ -1208,7 +1254,7 @@
:table)
;; Declared pipeline errors and refusals are recoverable; anything else ends the turn.
structured (:structured-output (tools/with-pipeline-errors
- (execute-representations-query query)))
+ (execute-representations-query query {:preprocess-check? true})))
;; A query built only to be read needs run_query, so without it the chart is the one way
;; the user sees anything and is always shown.
show-chart? (not (and (false? show_chart) (shared/tool-offered? "run_query")))]
diff --git a/src/metabase/metabot/tools/recoverable/pipeline.clj b/src/metabase/metabot/tools/recoverable/pipeline.clj
index 3def00dd7f2..bd0c136a47e 100644
--- a/src/metabase/metabot/tools/recoverable/pipeline.clj
+++ b/src/metabase/metabot/tools/recoverable/pipeline.clj
@@ -376,6 +376,11 @@
"The query parsed but would not run — the editor's own run gate rejects it."
[])
+(defpipeline-error query-preprocessing-failed!
+ "The query processor rejected the constructed query while preparing it to run."
+ [{:uses #{}
+ :text "Fix that part of the query and retry."}])
+
(defpipeline-error not-implemented-yet!
"The query used a representations feature the pipeline does not support yet."
;; No step: nothing the agent does to this query will make the feature exist. It can still make
diff --git a/src/metabase/metabot/tools/slackbot_query.clj b/src/metabase/metabot/tools/slackbot_query.clj
index efbe6303499..9806311ae1b 100644
--- a/src/metabase/metabot/tools/slackbot_query.clj
+++ b/src/metabase/metabot/tools/slackbot_query.clj
@@ -56,7 +56,7 @@
(handle [_ {:keys [query title display]} _ctx]
(let [structured (:structured-output (tools/with-pipeline-errors
- (construct/execute-representations-query query)))
+ (construct/execute-representations-query query {:preprocess-check? true})))
viz (cond-> {:query (:query structured)
:link (streaming/query->question-url (:query structured) display)}
title (assoc :title title)
diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.clj
index bd428504a94..a967923042d 100644
--- a/test/metabase/metabot/tools/construct_representations_test.clj
+++ b/test/metabase/metabot/tools/construct_representations_test.clj
@@ -34,16 +34,19 @@
[metabase.metabot.tools.recovery-hints :as v1-hints]
[metabase.metabot.tools.resources :as resources]
[metabase.metabot.tools.runtime :as tools.runtime]
+ [metabase.metabot.tools.slackbot-query :as slackbot-query]
[metabase.metabot.tools.util :as metabot.tools.util]
[metabase.models.interface :as mi]
[metabase.models.serialization.resolve :as serdes.resolve]
[metabase.models.serialization.resolve.mp :as resolve.mp]
[metabase.permissions.core :as perms]
[metabase.permissions.models.permissions-group :as perms-group]
- [metabase.query-processor :as qp]
+ [metabase.query-processor.core :as qp]
+ [metabase.query-processor.middleware.add-implicit-clauses :as qp.add-implicit-clauses]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util.json :as json]
+ [metabase.util.log.capture :as log.capture]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2])
(:import
@@ -3355,6 +3358,130 @@
(testing "an explanation with no errors falls back to a generic sentence and the usual causes"
(is (str/starts-with? (explain) "the query does not match the query builder's schema. This usually means")))))
+(deftest construct-tools-reject-invalid-filter-literals-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (doseq [[field-name value code message]
+ [["CREATED_AT" "2024-13-45" :invalid-temporal-literal #"(?s).*2024-13-45.*"]
+ ["TOTAL" "abc" :query-preprocessing-failed #"(?s).*unable to parse String.*"]]
+ tool [construct/construct-notebook-query-tool
+ slackbot-query/slackbot-construct-notebook-query-tool]]
+ (testing (str (type tool) " " field-name)
+ (let [db-name (:name (mt/db))
+ query {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql"
+ :source-table [db-name "PUBLIC" "ORDERS"]
+ :filters [["=" {} ["field" {} [db-name "PUBLIC" "ORDERS" field-name]]
+ value]]}]}]
+ (when (= code :query-preprocessing-failed)
+ (let [built (get-in (construct/execute-representations-query query) [:structured-output :query])]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo message (qp/preprocess built)))))
+ (is (=? {:class :recoverable
+ :code (keyword "metabase.metabot.tools.recoverable.pipeline" (name code))
+ :text message}
+ (test-util/tool-failure tool {:query query :reasoning "filter orders"}))))))))
+
+(deftest preprocessing-check-is-opt-in-test
+ (with-mp-and-stubs!
+ (fn []
+ (let [seen (atom [])
+ query {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-table ["Sample" "PUBLIC" "ORDERS"]}]}]
+ (mt/with-dynamic-fn-redefs [qp/preprocess (fn [query]
+ (swap! seen conj query)
+ (lib/limit query 1))]
+ (construct/execute-representations-query query)
+ (is (empty? @seen))
+ (let [built (construct/execute-representations-query query {:preprocess-check? true})]
+ (is (= 1 (count @seen)))
+ (is (= (first @seen) (get-in built [:structured-output :query])))))))))
+
+(deftest unrecognized-preprocessing-failures-allow-construction-test
+ (with-mp-and-stubs!
+ (fn []
+ (let [query {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-table ["Sample" "PUBLIC" "ORDERS"]}]}
+ wrap (fn [cause] (ex-info "Error preprocessing query in middleware: private detail"
+ {:type :qp :fn #'qp/preprocess} cause))
+ forbidden (ex-info "private permission detail" {:status-code 403})]
+ (doseq [e [(wrap (ex-info "another wrapper" {:type :invalid-query} forbidden))
+ (ex-info "private setup failure" {})
+ (wrap (RuntimeException. "private programming failure"))
+ (wrap (ex-info "private DB failure" {:type :invalid-query}
+ (java.sql.SQLException. "private JDBC detail")))
+ (wrap (ex-info "private app DB failure"
+ {:type :invalid-query :toucan2/context-trace []}))]
+ tool [construct/construct-notebook-query-tool
+ slackbot-query/slackbot-construct-notebook-query-tool]]
+ (mt/with-dynamic-fn-redefs [qp/preprocess (fn [_] (throw e))]
+ (log.capture/with-log-messages-for-level [logs [metabase.metabot.tools.construct :warn]]
+ (let [result (test-util/call-tool tool {:query query :reasoning "count orders"})]
+ (is (map? (get-in result [:structured-output :query])))
+ (is (not (str/includes? (pr-str result) "private"))))
+ (is (= [e] (map :e (logs)))))))))))
+
+(deftest preprocessing-rejections-quote-bounded-diagnostics-test
+ (with-mp-and-stubs!
+ (fn []
+ (let [long-diagnostic (str "Bad literal: " (apply str (repeat 400 "\u00e9")))
+ boundary-prefix (apply str (repeat 299 "x"))
+ query {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-table ["Sample" "PUBLIC" "ORDERS"]}]}]
+ (doseq [[diagnostic quoted-diagnostic]
+ [[long-diagnostic (str "\"" (subs long-diagnostic 0 300) "\"")]
+ ["Bad literal:\n\"abc\"" "\"Bad literal:\\n\\\"abc\\\"\""]
+ [(str boundary-prefix "\n\"tail") (str "\"" boundary-prefix "\\n\"")]]
+ error-type [:invalid-query :unsupported-feature]
+ tool [construct/construct-notebook-query-tool
+ slackbot-query/slackbot-construct-notebook-query-tool]]
+ (mt/with-dynamic-fn-redefs [qp/preprocess (fn [_]
+ (throw (ex-info "Error preprocessing query in middleware"
+ {:type :qp :fn #'qp/preprocess}
+ (ex-info diagnostic {:type error-type}))))]
+ (let [{:keys [class code text]} (test-util/tool-failure tool {:query query :reasoning "count orders"})]
+ (is (= :recoverable class))
+ (is (= :metabase.metabot.tools.recoverable.pipeline/query-preprocessing-failed code))
+ (is (= (str "Metabase could not prepare this query to run: "
+ quoted-diagnostic "\nFix that part of the query and retry.")
+ text)))))))))
+
+(deftest preprocessing-permission-failure-allows-construction-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [forbidden (ex-info "private permission detail" {:status-code 403})
+ query {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql"
+ :source-table [(:name (mt/db)) "PUBLIC" "ORDERS"]}]}]
+ (mt/with-dynamic-fn-redefs [qp.add-implicit-clauses/add-implicit-clauses (fn [_] (throw forbidden))]
+ (is (map? (get-in (construct/execute-representations-query query {:preprocess-check? true})
+ [:structured-output :query])))))))
+
+(deftest construct-tools-reject-misplaced-metrics-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [mp (mt/metadata-provider)
+ metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :products)))
+ (lib/aggregate (lib/sum (lib.metadata/field mp (mt/id :products :price)))))]
+ (mt/with-temp [:model/Card {entity-id :entity_id} {:name "Products price"
+ :type :metric
+ :database_id (mt/id)
+ :dataset_query (lib/prepare-for-serialization metric-query)}]
+ (doseq [[stages message]
+ [[[{:lib/type "mbql.stage/mbql"
+ :source-table [(:name (mt/db)) "PUBLIC" "ORDERS"]}
+ {:lib/type "mbql.stage/mbql"
+ :aggregation [["metric" {} entity-id]]}]
+ #"(?s).*run: A metric is incompatible.*stage's source.*Fix that part.*"]
+ [[{:lib/type "mbql.stage/mbql"
+ :source-table [(:name (mt/db)) "PUBLIC" "PRODUCTS"]
+ :aggregation [["count" {}]]
+ :filters [[">" {} ["metric" {} entity-id] 100]]}]
+ #"(?s).*run: Metabase could not expand a metric reference.*in an aggregation.*Fix that part.*"]]
+ tool [construct/construct-notebook-query-tool
+ slackbot-query/slackbot-construct-notebook-query-tool]]
+ (is (=? {:class :recoverable
+ :code :metabase.metabot.tools.recoverable.pipeline/query-preprocessing-failed
+ :text message}
+ (test-util/tool-failure tool {:query {:lib/type "mbql/query" :stages stages}
+ :reasoning "use a metric"}))))))))
+
(deftest structural-validation-locations-end-to-end-test
(with-mp-and-stubs!
(fn []
diff --git a/test/metabase/metabot/tools/slackbot_query_test.clj b/test/metabase/metabot/tools/slackbot_query_test.clj
index 211d1753303..12f5a3d251b 100644
--- a/test/metabase/metabot/tools/slackbot_query_test.clj
+++ b/test/metabase/metabot/tools/slackbot_query_test.clj
@@ -128,7 +128,7 @@
from its declaration in `tools.recoverable.pipeline`, filtered by the tools
the calling profile has — passing a hint as well would append a second copy
of the same advice."
- (is (nil? @captured-opts)))
+ (is (= {:preprocess-check? true} @captured-opts)))
(testing "structured-output is returned upstream"
(is (= "q-1" (get-in result [:structured-output :query-id])))
(is (= fake-query (get-in result [:structured-output :query]))))