The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Check queries before Metabot reports a chart as ready
Andrei Simionescu7e35c0c073e3
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.cljindex 0ae23a61295..ba5e69205ec 100644--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj@@ -3,18 +3,25 @@[clojure.string :as str][clojure.test :refer [deftest is testing]][metabase-enterprise.sandbox.test-util :as sandbox.tu]+ [metabase-enterprise.test :as met][metabase.api.common :as api][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.metabot.metadata-perms :as metabot.perms]+ [metabase.metabot.test-util :as test-util][metabase.metabot.tools.construct :as construct]+ [metabase.metabot.tools.slackbot-query :as slackbot-query][metabase.permissions.core :as perms][metabase.query-processor :as qp]+ [metabase.query-processor.compile :as qp.compile][metabase.query-processor.pipeline :as qp.pipeline][metabase.test :as mt][metabase.util :as u]+ [metabase.util.log.capture :as log.capture][toucan2.core :as t2]))+(set! *warn-on-reflection* true)+(defn- sandbox-query [table fields](let [mp (mt/metadata-provider)](lib/with-fields (lib/query mp (lib.metadata/table mp (mt/id table)))@@ -106,3 +113,79 @@(is (= :completed (:status actual-result)))(is (seq (mt/rows actual-result)))(is (= (set (mt/rows control-result)) (set (mt/rows actual-result)))))))))++(defn- count-query [table]+ {:lib/type "mbql/query"+ :stages [{:lib/type "mbql.stage/mbql"+ :source-table ["Metabot sandbox" "PUBLIC" table]+ :aggregation [["count" {}]]}]})++(defn- orders-sandbox-query []+ (let [mp (mt/metadata-provider)]+ (lib/filter (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/> (lib.metadata/field mp (mt/id :orders :total)) 10))))++(defn- check-sandbox-attribute-error! [attributes message]+ (met/with-gtaps! {:gtaps {:venues {:query (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/table mp (mt/id :venues))))+ :remappings {:cat ["variable" [:field (mt/id :venues :category_id) nil]]}}}+ :attributes attributes}+ (t2/update! :model/Database (mt/id) {:name "Metabot sandbox"})+ (let [query (count-query "VENUES")]+ (doseq [tool [construct/construct-notebook-query-tool+ slackbot-query/slackbot-construct-notebook-query-tool]]+ (let [built (get-in (test-util/call-tool tool {:query query :reasoning "count venues"})+ [:structured-output :query])]+ (is (map? built))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo message (qp/process-query built))))))))++(deftest construct-tools-allow-missing-sandbox-attribute-test+ (check-sandbox-attribute-error! {"something_random" 50} #"Query requires user attribute `cat`"))++(deftest construct-tools-preprocess-sandboxed-queries-test+ (met/with-gtaps! {:gtaps {:orders {:query (orders-sandbox-query)}}}+ (t2/update! :model/Database (mt/id) {:name "Metabot sandbox"})+ (let [compilations (atom 0)+ executions (atom 0)+ query (count-query "ORDERS")]+ (mt/with-dynamic-fn-redefs [qp.compile/compile-preprocessed (fn [& _]+ (swap! compilations inc)+ (throw (ex-info "Unexpected SQL compilation" {})))]+ (binding [qp.pipeline/*execute* (fn [& _]+ (swap! executions inc)+ (throw (ex-info "Unexpected query execution" {})))]+ (log.capture/with-log-messages-for-level [logs [metabase.metabot.tools.construct :warn]]+ (doseq [tool [construct/construct-notebook-query-tool+ slackbot-query/slackbot-construct-notebook-query-tool]]+ (let [built (get-in (test-util/call-tool tool {:query query :reasoning "count orders"})+ [:structured-output :query])]+ (is (= 1 (lib/stage-count built)))+ (is (= (mt/id :orders) (get-in built [:stages 0 :source-table])))))+ (is (empty? (logs)))))+ (is (zero? @compilations))+ (is (zero? @executions))))))++(deftest construct-tools-allow-invalid-sandbox-attribute-test+ (check-sandbox-attribute-error! {"cat" "a"} #"User attribute `cat` value `a` cannot be coerced"))++(deftest construct-tools-load-native-sandbox-metadata-test+ (doseq [tool [construct/construct-notebook-query-tool+ slackbot-query/slackbot-construct-notebook-query-tool]]+ (met/with-gtaps! {:gtaps {:venues {:query (lib/native-query+ (mt/metadata-provider)+ "SELECT ID, NAME, CATEGORY_ID, PRICE FROM VENUES WHERE PRICE = 1")}}}+ (t2/update! :model/Database (mt/id) {:name "Metabot sandbox"})+ (let [card-id (t2/select-one-fn :card_id :model/Sandbox :table_id (mt/id :venues))+ executions (atom 0)+ execute qp.pipeline/*execute*+ query (count-query "VENUES")]+ (is (nil? (t2/select-one-fn :result_metadata :model/Card :id card-id)))+ (binding [qp.pipeline/*execute* (fn [driver query respond]+ (swap! executions inc)+ (execute driver query respond))]+ (doseq [expected-executions [1 0]]+ (reset! executions 0)+ (is (map? (get-in (test-util/call-tool tool {:query query :reasoning "count venues"})+ [:structured-output :query])))+ (is (= expected-executions @executions))+ (is (seq (t2/select-one-fn :result_metadata :model/Card :id card-id)))))))))diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.cljindex a0ec40341f9..566291c7359 100644--- a/src/metabase/metabot/tools/construct.clj+++ b/src/metabase/metabot/tools/construct.clj@@ -36,9 +36,11 @@[metabase.query-processor.core :as qp][metabase.util :as u][metabase.util.json :as json]+ [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.match :as match])+ [metabase.util.match :as match]+ [metabase.util.quoting :as quoting])(:import(java.sql SQLException)))@@ -1098,18 +1100,62 @@(ex-info (str (ex-message e) "\n" hint) (ex-data e) e)e))+(def ^:private preprocessing-message-limit 300)++(defn- preprocessing-rejection-message+ [cause]+ (if (contains? #{:invalid-query :unsupported-feature} (:type (ex-data cause)))+ (quoting/quoted (u/truncate (or (ex-message cause) "Invalid query.") preprocessing-message-limit))+ (when (instance? clojure.lang.ExceptionInfo cause)+ (case (ex-message cause)+ "Incompatible metric"+ (str "A metric is incompatible with this query stage. Use a metric that applies to the stage's "+ "source table and columns.")++ "Failed to replace metric"+ (str "Metabase could not expand a metric reference. Check that it is in an aggregation and applies "+ "to this stage's source.")++ nil))))++(defn- check-query-preprocessing!+ [query]+ (try+ (qp/preprocess query)+ (catch Throwable e+ (let [chain (u/full-exception-chain e)+ message (preprocessing-rejection-message (ex-cause e))]+ (if (and (= :qp (:type (ex-data e)))+ (:fn (ex-data e))+ message+ (not-any? #(or (= 403 (:status-code (ex-data %)))+ (instance? java.sql.SQLException %)+ (contains? (ex-data %) :toucan2/context-trace)) chain))+ (throw (ex-info (str "Metabase could not prepare this query to run: "+ message)+ {:agent-error? true+ :error :query-preprocessing-failed+ :status-code 400}+ e))+ (log/warn e "Could not preprocess the constructed Metabot query"))))))+(defn execute-representations-query"Run `external-query` through the representations pipeline. See[[execute-representations-query*]] for the pipeline itself.`opts` may carry `:recovery-hint`, a function from an agent error's `ex-data` to a sentencestring (or nil) telling the caller's agent how to recover; the sentence is appended to the- agent error's message. Omitting it leaves agent errors with the pipeline's bare statement."+ agent error's message. Omitting it leaves agent errors with the pipeline's bare statement.++ `:preprocess-check?` checks the built query before returning it. Known query errors become+ agent errors; other preprocessing failures are logged and ignored."([external-query](execute-representations-query external-query nil))- ([external-query {:keys [recovery-hint]}]+ ([external-query {:keys [recovery-hint preprocess-check?]}](try- (execute-representations-query* external-query)+ (u/prog1 (execute-representations-query* external-query)+ (when preprocess-check?+ (check-query-preprocessing! (get-in <> [:structured-output :query]))))(catch clojure.lang.ExceptionInfo e(throw (if (:agent-error? (ex-data e))(with-recovery-hint e recovery-hint)@@ -1208,7 +1254,7 @@:table);; Declared pipeline errors and refusals are recoverable; anything else ends the turn.structured (:structured-output (tools/with-pipeline-errors- (execute-representations-query query)))+ (execute-representations-query query {:preprocess-check? true})));; A query built only to be read needs run_query, so without it the chart is the one way;; the user sees anything and is always shown.show-chart? (not (and (false? show_chart) (shared/tool-offered? "run_query")))]diff --git a/src/metabase/metabot/tools/recoverable/pipeline.clj b/src/metabase/metabot/tools/recoverable/pipeline.cljindex 3def00dd7f2..bd0c136a47e 100644--- a/src/metabase/metabot/tools/recoverable/pipeline.clj+++ b/src/metabase/metabot/tools/recoverable/pipeline.clj@@ -376,6 +376,11 @@"The query parsed but would not run — the editor's own run gate rejects it."[])+(defpipeline-error query-preprocessing-failed!+ "The query processor rejected the constructed query while preparing it to run."+ [{:uses #{}+ :text "Fix that part of the query and retry."}])+(defpipeline-error not-implemented-yet!"The query used a representations feature the pipeline does not support yet.";; No step: nothing the agent does to this query will make the feature exist. It can still makediff --git a/src/metabase/metabot/tools/slackbot_query.clj b/src/metabase/metabot/tools/slackbot_query.cljindex efbe6303499..9806311ae1b 100644--- a/src/metabase/metabot/tools/slackbot_query.clj+++ b/src/metabase/metabot/tools/slackbot_query.clj@@ -56,7 +56,7 @@(handle [_ {:keys [query title display]} _ctx](let [structured (:structured-output (tools/with-pipeline-errors- (construct/execute-representations-query query)))+ (construct/execute-representations-query query {:preprocess-check? true})))viz (cond-> {:query (:query structured):link (streaming/query->question-url (:query structured) display)}title (assoc :title title)diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.cljindex bd428504a94..a967923042d 100644--- a/test/metabase/metabot/tools/construct_representations_test.clj+++ b/test/metabase/metabot/tools/construct_representations_test.clj@@ -34,16 +34,19 @@[metabase.metabot.tools.recovery-hints :as v1-hints][metabase.metabot.tools.resources :as resources][metabase.metabot.tools.runtime :as tools.runtime]+ [metabase.metabot.tools.slackbot-query :as slackbot-query][metabase.metabot.tools.util :as metabot.tools.util][metabase.models.interface :as mi][metabase.models.serialization.resolve :as serdes.resolve][metabase.models.serialization.resolve.mp :as resolve.mp][metabase.permissions.core :as perms][metabase.permissions.models.permissions-group :as perms-group]- [metabase.query-processor :as qp]+ [metabase.query-processor.core :as qp]+ [metabase.query-processor.middleware.add-implicit-clauses :as qp.add-implicit-clauses][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util.json :as json]+ [metabase.util.log.capture :as log.capture][metabase.util.malli.registry :as mr][toucan2.core :as t2])(:import@@ -3355,6 +3358,130 @@(testing "an explanation with no errors falls back to a generic sentence and the usual causes"(is (str/starts-with? (explain) "the query does not match the query builder's schema. This usually means")))))+(deftest construct-tools-reject-invalid-filter-literals-test+ (mt/with-current-user (mt/user->id :crowberto)+ (doseq [[field-name value code message]+ [["CREATED_AT" "2024-13-45" :invalid-temporal-literal #"(?s).*2024-13-45.*"]+ ["TOTAL" "abc" :query-preprocessing-failed #"(?s).*unable to parse String.*"]]+ tool [construct/construct-notebook-query-tool+ slackbot-query/slackbot-construct-notebook-query-tool]]+ (testing (str (type tool) " " field-name)+ (let [db-name (:name (mt/db))+ query {:lib/type "mbql/query"+ :stages [{:lib/type "mbql.stage/mbql"+ :source-table [db-name "PUBLIC" "ORDERS"]+ :filters [["=" {} ["field" {} [db-name "PUBLIC" "ORDERS" field-name]]+ value]]}]}]+ (when (= code :query-preprocessing-failed)+ (let [built (get-in (construct/execute-representations-query query) [:structured-output :query])]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo message (qp/preprocess built)))))+ (is (=? {:class :recoverable+ :code (keyword "metabase.metabot.tools.recoverable.pipeline" (name code))+ :text message}+ (test-util/tool-failure tool {:query query :reasoning "filter orders"}))))))))++(deftest preprocessing-check-is-opt-in-test+ (with-mp-and-stubs!+ (fn []+ (let [seen (atom [])+ query {:lib/type "mbql/query"+ :stages [{:lib/type "mbql.stage/mbql" :source-table ["Sample" "PUBLIC" "ORDERS"]}]}]+ (mt/with-dynamic-fn-redefs [qp/preprocess (fn [query]+ (swap! seen conj query)+ (lib/limit query 1))]+ (construct/execute-representations-query query)+ (is (empty? @seen))+ (let [built (construct/execute-representations-query query {:preprocess-check? true})]+ (is (= 1 (count @seen)))+ (is (= (first @seen) (get-in built [:structured-output :query])))))))))++(deftest unrecognized-preprocessing-failures-allow-construction-test+ (with-mp-and-stubs!+ (fn []+ (let [query {:lib/type "mbql/query"+ :stages [{:lib/type "mbql.stage/mbql" :source-table ["Sample" "PUBLIC" "ORDERS"]}]}+ wrap (fn [cause] (ex-info "Error preprocessing query in middleware: private detail"+ {:type :qp :fn #'qp/preprocess} cause))+ forbidden (ex-info "private permission detail" {:status-code 403})]+ (doseq [e [(wrap (ex-info "another wrapper" {:type :invalid-query} forbidden))+ (ex-info "private setup failure" {})+ (wrap (RuntimeException. "private programming failure"))+ (wrap (ex-info "private DB failure" {:type :invalid-query}+ (java.sql.SQLException. "private JDBC detail")))+ (wrap (ex-info "private app DB failure"+ {:type :invalid-query :toucan2/context-trace []}))]+ tool [construct/construct-notebook-query-tool+ slackbot-query/slackbot-construct-notebook-query-tool]]+ (mt/with-dynamic-fn-redefs [qp/preprocess (fn [_] (throw e))]+ (log.capture/with-log-messages-for-level [logs [metabase.metabot.tools.construct :warn]]+ (let [result (test-util/call-tool tool {:query query :reasoning "count orders"})]+ (is (map? (get-in result [:structured-output :query])))+ (is (not (str/includes? (pr-str result) "private"))))+ (is (= [e] (map :e (logs)))))))))))++(deftest preprocessing-rejections-quote-bounded-diagnostics-test+ (with-mp-and-stubs!+ (fn []+ (let [long-diagnostic (str "Bad literal: " (apply str (repeat 400 "\u00e9")))+ boundary-prefix (apply str (repeat 299 "x"))+ query {:lib/type "mbql/query"+ :stages [{:lib/type "mbql.stage/mbql" :source-table ["Sample" "PUBLIC" "ORDERS"]}]}]+ (doseq [[diagnostic quoted-diagnostic]+ [[long-diagnostic (str "\"" (subs long-diagnostic 0 300) "\"")]+ ["Bad literal:\n\"abc\"" "\"Bad literal:\\n\\\"abc\\\"\""]+ [(str boundary-prefix "\n\"tail") (str "\"" boundary-prefix "\\n\"")]]+ error-type [:invalid-query :unsupported-feature]+ tool [construct/construct-notebook-query-tool+ slackbot-query/slackbot-construct-notebook-query-tool]]+ (mt/with-dynamic-fn-redefs [qp/preprocess (fn [_]+ (throw (ex-info "Error preprocessing query in middleware"+ {:type :qp :fn #'qp/preprocess}+ (ex-info diagnostic {:type error-type}))))]+ (let [{:keys [class code text]} (test-util/tool-failure tool {:query query :reasoning "count orders"})]+ (is (= :recoverable class))+ (is (= :metabase.metabot.tools.recoverable.pipeline/query-preprocessing-failed code))+ (is (= (str "Metabase could not prepare this query to run: "+ quoted-diagnostic "\nFix that part of the query and retry.")+ text)))))))))++(deftest preprocessing-permission-failure-allows-construction-test+ (mt/with-current-user (mt/user->id :crowberto)+ (let [forbidden (ex-info "private permission detail" {:status-code 403})+ query {:lib/type "mbql/query"+ :stages [{:lib/type "mbql.stage/mbql"+ :source-table [(:name (mt/db)) "PUBLIC" "ORDERS"]}]}]+ (mt/with-dynamic-fn-redefs [qp.add-implicit-clauses/add-implicit-clauses (fn [_] (throw forbidden))]+ (is (map? (get-in (construct/execute-representations-query query {:preprocess-check? true})+ [:structured-output :query])))))))++(deftest construct-tools-reject-misplaced-metrics-test+ (mt/with-current-user (mt/user->id :crowberto)+ (let [mp (mt/metadata-provider)+ metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :products)))+ (lib/aggregate (lib/sum (lib.metadata/field mp (mt/id :products :price)))))]+ (mt/with-temp [:model/Card {entity-id :entity_id} {:name "Products price"+ :type :metric+ :database_id (mt/id)+ :dataset_query (lib/prepare-for-serialization metric-query)}]+ (doseq [[stages message]+ [[[{:lib/type "mbql.stage/mbql"+ :source-table [(:name (mt/db)) "PUBLIC" "ORDERS"]}+ {:lib/type "mbql.stage/mbql"+ :aggregation [["metric" {} entity-id]]}]+ #"(?s).*run: A metric is incompatible.*stage's source.*Fix that part.*"]+ [[{:lib/type "mbql.stage/mbql"+ :source-table [(:name (mt/db)) "PUBLIC" "PRODUCTS"]+ :aggregation [["count" {}]]+ :filters [[">" {} ["metric" {} entity-id] 100]]}]+ #"(?s).*run: Metabase could not expand a metric reference.*in an aggregation.*Fix that part.*"]]+ tool [construct/construct-notebook-query-tool+ slackbot-query/slackbot-construct-notebook-query-tool]]+ (is (=? {:class :recoverable+ :code :metabase.metabot.tools.recoverable.pipeline/query-preprocessing-failed+ :text message}+ (test-util/tool-failure tool {:query {:lib/type "mbql/query" :stages stages}+ :reasoning "use a metric"}))))))))+(deftest structural-validation-locations-end-to-end-test(with-mp-and-stubs!(fn []diff --git a/test/metabase/metabot/tools/slackbot_query_test.clj b/test/metabase/metabot/tools/slackbot_query_test.cljindex 211d1753303..12f5a3d251b 100644--- a/test/metabase/metabot/tools/slackbot_query_test.clj+++ b/test/metabase/metabot/tools/slackbot_query_test.clj@@ -128,7 +128,7 @@from its declaration in `tools.recoverable.pipeline`, filtered by the toolsthe calling profile has — passing a hint as well would append a second copyof the same advice."- (is (nil? @captured-opts)))+ (is (= {:preprocess-check? true} @captured-opts)))(testing "structured-output is returned upstream"(is (= "q-1" (get-in result [:structured-output :query-id])))(is (= fake-query (get-in result [:structured-output :query]))))