The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Postgres mbql5 compilation (#72573)
Riley Thompson61d3fa771f63
diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.cljindex 548f7371bec..98e78b37f00 100644--- a/src/metabase/driver/h2.clj+++ b/src/metabase/driver/h2.clj@@ -102,7 +102,7 @@(defmethod sql.qp/->honeysql [:h2-mbql5 :regex-match-first][driver [_ _opts arg pattern]]- [:regexp_substr (sql.qp/->honeysql driver arg) (sql.qp/->honeysql driver pattern)])+ ((get-method sql.qp/->honeysql [:h2 :regex-match-first]) driver [:regex-match-first arg pattern]))(defmethod driver/connection-properties :h2[_]@@ -435,7 +435,7 @@(defmethod sql.qp/->honeysql [:h2-mbql5 :log][driver [_ _opts field]]- [:log10 (sql.qp/->honeysql driver field)])+ ((get-method sql.qp/->honeysql [:h2 :log]) driver [:log field]))(defmethod sql.qp/->honeysql [:h2 ::sql.qp/expression-literal-text-value][driver [_ value]]@@ -450,14 +450,7 @@(defmethod sql.qp/->honeysql [:h2-mbql5 ::sql.qp/expression-literal-text-value][driver [_ _opts value]]- ;; A literal text value gets compiled to a parameter placeholder like "?". H2 attempts to compile the prepared- ;; statement immediately, presumably before the types of the params are known, and sometimes raises an "Unknown- ;; data type" error if it can't deduce the type. The recommended workaround is to insert an explicit CAST.- ;;- ;; https://linear.app/metabase/issue/QUE-726/- ;; https://github.com/h2database/h2database/issues/1383- (->> (sql.qp/->honeysql driver value)- (h2x/cast :text)))+ ((get-method sql.qp/->honeysql [:h2 ::sql.qp/expression-literal-text-value]) driver [::sql.qp/expression-literal-text-value value]))(defn- datediff"Like H2's `datediff` function but accounts for timestamps with time zones."diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.cljindex 04acdc4a8e4..922c72d686a 100644--- a/src/metabase/driver/postgres.clj+++ b/src/metabase/driver/postgres.clj@@ -61,6 +61,7 @@;; redundant *and* the literal `'\'` is unparseable by the PG JDBC driver when the server has;; `standard_conforming_strings = off` (#73721).(driver/register! :postgres, :parent #{:sql-jdbc ::like-escape-char-built-in/like-escape-char-built-in})+(driver/register! :postgres-mbql5, :parent #{:postgres :sql-mbql5})(defmethod driver/display-name :postgres [_] "PostgreSQL")@@ -110,7 +111,7 @@:database-replication]](defmethod driver/database-supports? [:postgres feature][driver _feat _db]- (= driver :postgres)))+ (or (= driver :postgres) (= driver :postgres-mbql5))))(defmethod driver/escape-entity-name-for-metadata :postgres [_driver entity-name](when entity-name@@ -608,22 +609,37 @@expr)]](h2x/with-database-type-info expr "timestamp")))+(defmethod sql.qp/->honeysql [:postgres-mbql5 :convert-timezone]+ [driver [_ _opts arg target-timezone source-timezone]]+ ((get-method sql.qp/->honeysql [:postgres :convert-timezone])+ driver [:convert-timezone arg target-timezone source-timezone]))+(defmethod sql.qp/->honeysql [:postgres :value]- [driver value]- (let [[_ raw-value {base-type :base_type, database-type :database_type}] value]- (when (some? raw-value)- (condp #(isa? %2 %1) base-type- :type/PostgresBitString (h2x/cast :varbit raw-value)- :type/IPAddress (h2x/cast :inet raw-value)- :type/PostgresEnum (if (quoted? database-type)- (h2x/cast database-type raw-value)- (h2x/quoted-cast database-type raw-value))- ((get-method sql.qp/->honeysql [:sql-jdbc :value])- driver value)))))+ [driver [_ raw-value {base-type :base_type database-type :database_type}]]+ (when (some? raw-value)+ (condp #(isa? %2 %1) base-type+ :type/PostgresBitString (h2x/cast :varbit raw-value)+ :type/IPAddress (h2x/cast :inet raw-value)+ :type/PostgresEnum (if (quoted? database-type)+ (h2x/cast database-type raw-value)+ (h2x/quoted-cast database-type raw-value))+ ((get-method sql.qp/->honeysql [:sql-jdbc :value])+ driver [:value raw-value {:base_type base-type :database_type database-type}]))))++(defmethod sql.qp/->honeysql [:postgres-mbql5 :value]+ [driver [_ {:keys [base-type effective-type database-type]} raw-value]]+ ((get-method sql.qp/->honeysql [:postgres :value])+ driver [:value raw-value {:base_type base-type+ :effective_type effective-type+ :database_type database-type}]))(defmethod sql.qp/->honeysql [:postgres :median][driver [_ arg]]- (sql.qp/->honeysql driver [:percentile arg 0.5]))+ (sql.qp/->honeysql driver (sql.qp/mbql-clause driver :percentile arg 0.5)))++(defmethod sql.qp/->honeysql [:postgres-mbql5 :median]+ [driver [_ _opts arg]]+ ((get-method sql.qp/->honeysql [:postgres :median]) driver [:median arg]))(defmethod sql.qp/datetime-diff [:postgres :year][_driver _unit x y]@@ -681,6 +697,10 @@(let [identifier (sql.qp/->honeysql driver arg)][::regex-match-first identifier pattern]))+(defmethod sql.qp/->honeysql [:postgres-mbql5 :regex-match-first]+ [driver [_ _opts arg pattern]]+ ((get-method sql.qp/->honeysql [:postgres :regex-match-first]) driver [:regex-match-first arg pattern]))+(defmethod sql.qp/->honeysql [:postgres :split-part][driver [_ text divider position]](let [position (sql.qp/->honeysql driver position)]@@ -691,10 +711,18 @@:else[:split_part (sql.qp/->honeysql driver text) (sql.qp/->honeysql driver divider) position]]))+(defmethod sql.qp/->honeysql [:postgres-mbql5 :split-part]+ [driver [_ _opts text divider position]]+ ((get-method sql.qp/->honeysql [:postgres :split-part]) driver [:split-part text divider position]))+(defmethod sql.qp/->honeysql [:postgres :text][driver [_ value]](h2x/maybe-cast "TEXT" (sql.qp/->honeysql driver value)))+(defmethod sql.qp/->honeysql [:postgres-mbql5 :text]+ [driver [_ _opts value]]+ ((get-method sql.qp/->honeysql [:postgres :text]) driver [:text value]))+(defn- format-pg-conversion [_fn [expr psql-type]](let [[expr-sql & expr-args] (sql/format-expr expr {:nested true})](into [(format "%s::%s" expr-sql (name psql-type))]@@ -799,6 +827,10 @@:elseidentifier)))+(defmethod sql.qp/->honeysql [:postgres-mbql5 :field]+ [driver [_ opts id-or-name]]+ ((get-method sql.qp/->honeysql [:postgres :field]) driver [:field id-or-name opts]))+;; Postgres is not happy with JSON fields which are in group-bys or order-bys;; being described twice instead of using the alias.;; Therefore, force the alias, but only for JSON fields to avoid ambiguity.@@ -821,10 +853,28 @@(select-keys unqualified #{:group-by}))qualified)))+(defmethod sql.qp/apply-top-level-clause+ [:postgres-mbql5 :breakout]+ [driver clause honeysql-form {breakout-fields :breakout, _fields-fields :fields :as query}]+ (let [stored-field-ids (map last breakout-fields)+ stored-fields (map #(when (integer? %)+ (driver-api/field (driver-api/metadata-provider) %))+ stored-field-ids)+ parent-method (partial (get-method sql.qp/apply-top-level-clause [:sql :breakout])+ driver clause honeysql-form)+ qualified (parent-method query)+ unqualified (parent-method (update query+ :breakout+ #(sql.qp/rewrite-fields-to-force-using-column-aliases % {:is-breakout true})))]+ (if (some driver-api/json-field? stored-fields)+ (merge qualified+ (select-keys unqualified #{:group-by}))+ qualified)))+(defn- order-by-is-json-field?- [clause]- (let [is-aggregation? (= (-> clause (second) (first)) :aggregation)- stored-field-id (-> clause (second) (second))+ [clause n]+ (let [is-aggregation? (= (-> clause (nth n) (first)) :aggregation)+ stored-field-id (-> clause (nth n) (nth n))stored-field (when (and (not is-aggregation?) (integer? stored-field-id))(driver-api/field (driver-api/metadata-provider) stored-field-id))](and@@ -833,18 +883,34 @@(defmethod sql.qp/->honeysql [:postgres :desc][driver clause]- (let [new-clause (if (order-by-is-json-field? clause)+ (let [new-clause (if (order-by-is-json-field? clause 1)(sql.qp/rewrite-fields-to-force-using-column-aliases clause)clause)]((get-method sql.qp/->honeysql [:sql :desc]) driver new-clause)))+(defmethod sql.qp/->honeysql [:postgres-mbql5 :desc]+ [driver clause]+ (let [new-clause (if (order-by-is-json-field? clause 2)+ (sql.qp/rewrite-fields-to-force-using-column-aliases clause)+ clause)+ [_ opts ordered-clause] new-clause]+ ((get-method sql.qp/->honeysql [:sql :desc]) driver [:desc ordered-clause opts])))+(defmethod sql.qp/->honeysql [:postgres :asc][driver clause]- (let [new-clause (if (order-by-is-json-field? clause)+ (let [new-clause (if (order-by-is-json-field? clause 1)(sql.qp/rewrite-fields-to-force-using-column-aliases clause)clause)]((get-method sql.qp/->honeysql [:sql :asc]) driver new-clause)))+(defmethod sql.qp/->honeysql [:postgres-mbql5 :asc]+ [driver clause]+ (let [new-clause (if (order-by-is-json-field? clause 2)+ (sql.qp/rewrite-fields-to-force-using-column-aliases clause)+ clause)+ [_ opts ordered-clause] new-clause]+ ((get-method sql.qp/->honeysql [:sql :asc]) driver [:asc ordered-clause opts])))+;;; +----------------------------------------------------------------------------------------------------------------+;;; | metabase.driver.sql-jdbc impls |;;; +----------------------------------------------------------------------------------------------------------------+diff --git a/src/metabase/driver/postgres/actions.clj b/src/metabase/driver/postgres/actions.cljindex 97e5fcc0165..84e5cca9732 100644--- a/src/metabase/driver/postgres/actions.clj+++ b/src/metabase/driver/postgres/actions.clj@@ -99,6 +99,35 @@:message (tru "Some of your values violate the constraint: {0}" constraint-name):errors {}}))+(defn- maybe-parse-sql-error-with-postgres+ [driver error-type database action-type error-message]+ ((get-method sql-jdbc.actions/maybe-parse-sql-error [:postgres error-type])+ driver error-type database action-type error-message))++(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-not-null-constraint]+ [driver error-type database action-type error-message]+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))++(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-unique-constraint]+ [driver error-type database action-type error-message]+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))++(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-foreign-key-constraint]+ [driver error-type database action-type error-message]+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))++(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/incorrect-value-type]+ [driver error-type database action-type error-message]+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))++(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-permission-constraint]+ [driver error-type database action-type error-message]+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))++(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-check-constraint]+ [driver error-type database action-type error-message]+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))+(defmethod sql-jdbc.actions/base-type->sql-type-map :postgres[_driver]{:type/BigInteger "BIGINT"diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex f257af50ea3..9d115afcc8d 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -2266,7 +2266,8 @@mbql5-query (driver-api/query-from-legacy-inner-query metadata-provider database-id inner-query)](recur driver mbql5-query))))-(def ^:private experimental-mbql5-drivers {:h2 :h2-mbql5})+(def ^:private experimental-mbql5-drivers {:h2 :h2-mbql5+ :postgres :postgres-mbql5})(defn- mbql5-experiment-report[driver experimental-driver query]diff --git a/src/metabase/driver/sql/query_processor/util.clj b/src/metabase/driver/sql/query_processor/util.cljindex 52d666ab401..e7014cc4ca0 100644--- a/src/metabase/driver/sql/query_processor/util.clj+++ b/src/metabase/driver/sql/query_processor/util.clj@@ -1,5 +1,6 @@(ns metabase.driver.sql.query-processor.util(:require+ [metabase.driver-api.core :as driver-api][metabase.util.honey-sql-2 :as h2x]))(defn nfc-field->parent-identifier@@ -24,9 +25,11 @@(defn field-with-tz?"Given a clause, possibly a `:field` ref, return true iff its type is known to have timezone information."[arg]- (if-let [[kind _id-or-name opts] (and (vector? arg) arg)]- (and (= kind :field)- (when-let [type ((some-fn :effective-type :base-type) opts)]- (or (isa? type :type/DateTimeWithTZ)- (isa? type :type/TimeWithTZ))))+ (if-let [opts (driver-api/match-one arg+ [:field (opts :guard :lib/uuid) _] opts ;; mbql5+ [:field _ opts] opts ;; mbql4+ _ nil)]+ (when-let [type ((some-fn :effective-type :base-type) opts)]+ (or (isa? type :type/DateTimeWithTZ)+ (isa? type :type/TimeWithTZ)))false))diff --git a/src/metabase/driver/sql/util.clj b/src/metabase/driver/sql/util.cljindex 765c0fae3ee..21a6e718f2d 100644--- a/src/metabase/driver/sql/util.clj+++ b/src/metabase/driver/sql/util.clj@@ -162,6 +162,7 @@:n1ql Dialect/N1ql:plsql Dialect/PlSql:postgres Dialect/PostgreSql+ :postgres-mbql5 Dialect/PostgreSql:redshift Dialect/Redshift:sparksql Dialect/SparkSql:standardsql Dialect/StandardSqldiff --git a/src/metabase/driver/sql_jdbc/actions.clj b/src/metabase/driver/sql_jdbc/actions.cljindex 219bcd17baa..cc92c7ccd1d 100644--- a/src/metabase/driver/sql_jdbc/actions.clj+++ b/src/metabase/driver/sql_jdbc/actions.clj@@ -124,7 +124,7 @@(if-let [sql-type (type->sql-type base-type)](h2x/cast sql-type value)(try- (sql.qp/->honeysql driver (sql.qp/mbql-clause driver :value value field))+ (sql.qp/->honeysql driver (sql.qp/mbql-clause-with-opts driver :value field value))(catch Exception e(throw (ex-info (str "column cast failed: " (pr-str col-name)){:column col-namediff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 5088db1cd36..c9a130c3a3f 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -1,4 +1,4 @@-(ns metabase.driver.postgres-test+(ns ^:mb/driver-tests metabase.driver.postgres-test"Tests for features/capabilities specific to PostgreSQL driver, such as support for Postgres UUID or enum types."{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.postgres-test]}metabase.test.data/query {:namespaces [metabase.driver.postgres-test]}@@ -31,6 +31,7 @@[metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync.interface][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor-test-util :as sql.qp-test-util]+ [metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.test-metadata :as meta]@@ -50,6 +51,8 @@[metabase.sync.sync-metadata.tables :as sync-tables][metabase.sync.util :as sync-util][metabase.test :as mt]+ [metabase.test.data.datasets :as mtd]+ [metabase.test.data.env :as tx.env][metabase.test.data.interface :as tx][metabase.util :as u][metabase.util.honey-sql-2 :as h2x]@@ -65,6 +68,18 @@(set! *warn-on-reflection* true)+;; TODO: once h2-mbql5 is on master, share code with that+(defn- test-driver [driver thunk]+ (when (contains? (tx.env/test-drivers) driver)+ (testing (str "\n" driver "\n")+ (driver/with-driver (tx/the-driver-with-test-extensions driver)+ (thunk))+ ;; the above is the original definition of test-driver, but we add in+ ;; this clause to avoid having to rewrite all the tests below twice:+ (when (= driver :postgres)+ (driver/with-driver (tx/the-driver-with-test-extensions :postgres-mbql5)+ (thunk))))))+(use-fixtures :each (fn [thunk];; 1. If sync fails when loading a test dataset, don't swallow the error; throw an Exception so we;; can debug it. This is much less confusing when trying to fix broken tests.@@ -72,7 +87,11 @@;; 2. Make sure we're in Honey SQL 2 mode for all the little SQL snippets we're compiling in these;; tests.(binding [sync-util/*log-exceptions-and-continue?* false]- (mt/with-test-user :rasta (thunk)))))+ ;; NB: because of test parallelism, this *will* affect other non-pg+ ;; tests, but the check above in the test-driver function will+ ;; prevent it from actually doing anything different in those tests.+ (with-redefs [mtd/-test-driver test-driver]+ (mt/with-test-user :rasta (thunk))))))(deftest ^:parallel extract-test(is (= ["extract(month from NOW())"]@@ -433,7 +452,7 @@(mt/test-driver :postgres(testing "Deal with complicated identifier (#22967)"(qp.store/with-metadata-provider (lib.tu/mock-metadata-provider- {:database (assoc meta/database :engine :postgres, :id 1)+ {:database (assoc meta/database :engine driver/*driver*, :id 1):tables [(merge (meta/table-metadata :venues){:id 1:db-id 1@@ -444,19 +463,27 @@:table-id 1:nfc-path ["jsons" "values" "qty"]:database-type "integer"})]})- (let [field-clause [:field 1 {:binning- {:strategy :num-bins- :num-bins 100- :min-value 0.75- :max-value 54.0- :bin-width 0.75}}]]+ (let [field-clause (sql.qp/mbql-clause-with-opts driver/*driver*+ :field+ {:binning+ {:strategy :num-bins+ :num-bins 100+ :min-value 0.75+ :max-value 54.0+ :bin-width 0.75}}+ 1)](is (= ["((FLOOR((((complicated_identifiers.jsons#>> (array[?, ?]::text[]))::integer - 0.75) / 0.75)) * 0.75) + 0.75)""values" "qty"]- (sql/format-expr (sql.qp/->honeysql :postgres field-clause) {:nested true}))))))))+ (sql/format-expr (sql.qp/->honeysql driver/*driver* field-clause) {:nested true}))))))))++(defn- ^:private maybe-convert-and-compile [driver query]+ (cond-> query+ (= driver :postgres-mbql5) (lib.convert/->mbql5)+ :always qp.compile/compile))-(def ^:private json-alias-mock-metadata-provider+(defn- ^:private json-alias-mock-metadata-provider [driver](lib.tu/mock-metadata-provider- {:database (assoc meta/database :engine :postgres, :id 1)+ {:database (assoc meta/database :engine driver, :id 1):tables [(merge (meta/table-metadata :venues){:id 1:db-id 1@@ -474,21 +501,25 @@(deftest ^:parallel json-alias-test(mt/test-driver :postgres(testing "json breakouts and order bys have alias coercion"- (qp.store/with-metadata-provider json-alias-mock-metadata-provider- (let [field-bucketed [:field 1- {:temporal-unit :month- :metabase.query-processor.util.add-alias-info/source-table 1- :metabase.query-processor.util.add-alias-info/source-alias "dontwannaseethis"- :metabase.query-processor.util.add-alias-info/desired-alias "dontwannaseethis"- :metabase.query-processor.util.add-alias-info/position 1}]- compile-res (qp.compile/compile- (mt/query nil- {:database 1- :type :query- :query {:source-table 1- :aggregation [[:count]]- :breakout [field-bucketed]- :order-by [[:asc field-bucketed]]}}))]+ (qp.store/with-metadata-provider (json-alias-mock-metadata-provider driver/*driver*)+ ;; need to make this a function to avoid a duplicate `:lib/uuid` error when we use it in `compile-res`+ (let [field-bucketed-fn (fn []+ (sql.qp/mbql-clause-with-opts+ driver/*driver* :field+ {:temporal-unit :month+ :metabase.query-processor.util.add-alias-info/source-table 1+ :metabase.query-processor.util.add-alias-info/source-alias "dontwannaseethis"+ :metabase.query-processor.util.add-alias-info/desired-alias "dontwannaseethis"+ :metabase.query-processor.util.add-alias-info/position 1} 1))+ compile-res (maybe-convert-and-compile+ driver/*driver*+ (mt/query nil+ {:database 1+ :type :query+ :query {:source-table 1+ :aggregation [[:count]]+ :breakout [(field-bucketed-fn)]+ :order-by [[:asc (field-bucketed-fn)]]}}))](is (= ["SELECT"" DATE_TRUNC("" 'month',"@@ -503,7 +534,7 @@" \"json_alias_test\"""ORDER BY"" \"json_alias_test\" ASC"]- (str/split-lines (driver/prettify-native-form :postgres (:query compile-res)))))+ (str/split-lines (driver/prettify-native-form driver/*driver* (:query compile-res)))))(is (= ["injection' OR 1=1--' AND released = 1""injection' OR 1=1--' AND released = 1"](:params compile-res))))))))@@ -511,13 +542,14 @@(deftest ^:parallel json-alias-test-2(mt/test-driver :postgres(testing "json breakouts and order bys have alias coercion"- (qp.store/with-metadata-provider json-alias-mock-metadata-provider- (let [field-ordinary [:field 1 nil]- only-order (qp.compile/compile- {:database 1- :type :query- :query {:source-table 1- :order-by [[:asc field-ordinary]]}})]+ (qp.store/with-metadata-provider (json-alias-mock-metadata-provider driver/*driver*)+ (let [field-ordinary (sql.qp/mbql-clause-with-opts driver/*driver* :field nil 1)+ only-order (maybe-convert-and-compile+ driver/*driver*+ {:database 1+ :type :query+ :query {:source-table 1+ :order-by [[:asc field-ordinary]]}})](is (= ["SELECT"" (\"json_alias_test\".\"bob\" #>> (array [ ?, ? ] :: text [ ])) :: VARCHAR AS \"json_alias_test\"""FROM"@@ -526,11 +558,11 @@" \"json_alias_test\" ASC""LIMIT"" 1048575"]- (str/split-lines (driver/prettify-native-form :postgres (:query only-order))))))))))+ (str/split-lines (driver/prettify-native-form driver/*driver* (:query only-order))))))))))-(def ^:private json-alias-in-model-mock-metadata-provider+(defn- ^:private json-alias-in-model-mock-metadata-provider [driver](providers.mock/mock-metadata-provider- json-alias-mock-metadata-provider+ (json-alias-mock-metadata-provider driver){:cards [{:name "Model with JSON":id 123:database-id 1@@ -544,8 +576,9 @@(deftest ^:parallel json-breakout-in-model-test(mt/test-driver :postgres(testing "JSON columns in inner queries are referenced properly in outer queries #34930"- (qp.store/with-metadata-provider json-alias-in-model-mock-metadata-provider- (let [nested (qp.compile/compile+ (qp.store/with-metadata-provider (json-alias-in-model-mock-metadata-provider driver/*driver*)+ (let [nested (maybe-convert-and-compile+ driver/*driver*{:database (meta/id):type :query:query {:source-table "card__123"}})]@@ -898,7 +931,7 @@(testing "check that values for enum types get wrapped in appropriate CAST() fn calls in `->honeysql`"(is (= (h2x/with-database-type-info [:cast "toucan" (h2x/identifier :type-name "bird type")]"bird type")- (sql.qp/->honeysql :postgres [:value "toucan" {:database_type "bird type", :base_type :type/PostgresEnum}]))))))+ (sql.qp/->honeysql driver/*driver* (sql.qp/mbql-clause-with-opts driver/*driver* :value {:database_type "bird type", :base_type :type/PostgresEnum} "toucan")))))))(deftest enums-test-2(mt/test-driver :postgres@@ -2144,7 +2177,8 @@(deftest canceled-query-no-stacktrace-test(mt/test-driver :postgres(letfn [(catch-exceptions [run]- (let [query (merge {:type :query, :database 1} {})+ (let [query (cond-> {:type :query, :database 1}+ (= driver/*driver* :postgres-mbql5) (lib.convert/->mbql5))metadata {}rows []qp (fn [query rff]@@ -2153,7 +2187,7 @@(respond metadata rows))](qp.pipeline/*run* query rff)))qp (catch-exceptions/catch-exceptions qp)- result (driver/with-driver :h2+ result (driver/with-driver driver/*driver*(qp (qp/userland-query query) qp.reducible/default-rff))](cond-> result(map? result) (update :data dissoc :rows))))@@ -2174,7 +2208,7 @@;; Wrap it in an ExceptionInfo with the :query-canceled? flag, as our code does(catch-exceptions(fn [] (throw (ex-info "Error executing query: canceling statement due to user request"- {:driver :postgres+ {:driver driver/*driver*:sql ["SELECT pg_sleep(1000)"]:params []:type qp.error-type/invalid-query@@ -2183,16 +2217,19 @@(is (= 0 (count (into [] (cancel-messages) (log-messages))))"Query cancellation exceptions should not be logged")))- (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]- (future- (Thread/sleep 400)- (a/put! qp.pipeline/*canceled-chan* :cancel))- (mt/with-log-messages-for-level [messages :error]- (let [response (qp/process-query (assoc-in (mt/native-query {:query "select pg_sleep(8), false"})- [:middleware :userland-query?] true))]- (is (= "ERROR: canceling statement due to user request" (:error response)))- (let [bad-messages (into [] (cancel-messages) (messages))]- (is (empty? bad-messages)))))))))+ (let [mp (mt/metadata-provider)]+ ;; Refresh the permission set in case the metadata provider created this test DB.+ (mt/with-test-user :rasta+ (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]+ (future+ (Thread/sleep 400)+ (a/put! qp.pipeline/*canceled-chan* :cancel))+ (mt/with-log-messages-for-level [messages :error]+ (let [response (qp/process-query (assoc-in (lib/native-query mp "select pg_sleep(8), false")+ [:middleware :userland-query?] true))]+ (is (= "ERROR: canceling statement due to user request" (:error response)))+ (let [bad-messages (into [] (cancel-messages) (messages))]+ (is (empty? bad-messages)))))))))))(deftest bit-strings-can-be-filtered(mt/test-driver :postgres@@ -2263,28 +2300,32 @@(deftest set-network-timeout-test(mt/test-driver :postgres- (testing "network hangs are interrupted after *network-timeout-ms*"- (binding [driver.settings/*network-timeout-ms* 3000]- (is (thrown-with-msg?- org.postgresql.util.PSQLException- #"An I/O error occurred while sending to the backend"- (try- (sql-jdbc.execute/do-with-connection-with-options- driver/*driver* (mt/id) nil- (fn [^Connection conn]- (with-open [stmt (.createStatement conn)]- (.execute stmt "SELECT pg_sleep(6)"))))- (catch Exception e- (is (true? (some #(instance? java.net.SocketTimeoutException %)- (u/full-exception-chain e))))- (throw e)))))))- (testing "network hangs are not interrupted before *network-timeout-ms*"- (is (true?- (sql-jdbc.execute/do-with-connection-with-options- driver/*driver* (mt/id) nil- (fn [^Connection conn]- (with-open [stmt (.createStatement conn)]- (.execute stmt "SELECT pg_sleep(6)")))))))))+ (let [db-name (u/lower-case-en (format "network-timeout-%s-%s" (name driver/*driver*) (mt/random-name)))+ spec (sql-jdbc.conn/connection-details->spec+ driver/*driver*+ (mt/dbdef->connection-details driver/*driver* :db {:database-name db-name}))]+ (tx/with-temp-database! driver/*driver* db-name+ ;; Use a raw spec against a unique DB; the shared test-data DB can be created/dropped by concurrent+ ;; :postgres and :postgres-mbql5 test setup.+ (letfn [(run-pg-sleep []+ (sql-jdbc.execute/do-with-connection-with-options+ driver/*driver* spec nil+ (fn [^Connection conn]+ (with-open [stmt (.createStatement conn)]+ (.execute stmt "SELECT pg_sleep(6)")))))]+ (testing "network hangs are interrupted after *network-timeout-ms*"+ (binding [driver.settings/*network-timeout-ms* 3000]+ (is (thrown-with-msg?+ org.postgresql.util.PSQLException+ #"An I/O error occurred while sending to the backend"+ (try+ (run-pg-sleep)+ (catch Exception e+ (is (true? (some #(instance? java.net.SocketTimeoutException %)+ (u/full-exception-chain e))))+ (throw e)))))))+ (testing "network hangs are not interrupted before *network-timeout-ms*"+ (is (true? (run-pg-sleep)))))))))(deftest ^:parallel parse-final-identifier-test(mt/test-driver :postgresdiff --git a/test/metabase/driver/sql/parameters/substitution_test.clj b/test/metabase/driver/sql/parameters/substitution_test.cljindex 038639ea602..800ef34b02c 100644--- a/test/metabase/driver/sql/parameters/substitution_test.clj+++ b/test/metabase/driver/sql/parameters/substitution_test.clj@@ -16,17 +16,18 @@(set! *warn-on-reflection* true)-(deftest ^:parallel field->field-ref-test- (is (=? [:field- {:base-type :type/BigInteger- :metabase.query-processor.util.add-alias-info/source-table (meta/id :venues)- :metabase.driver.sql.parameters.substitution/compiling-field-filter? true}- (meta/id :venues :id)]- (#'sql.params.substitution/field->field-ref- :h2- (meta/field-metadata :venues :id)- :number/=- nil))))+(deftest ^:parallel field->field-filter-clause-test+ (mt/test-driver :h2+ (is (=? [:field+ {:base-type :type/BigInteger,+ :metabase.query-processor.util.add-alias-info/source-table (meta/id :venues),+ :metabase.driver.sql.parameters.substitution/compiling-field-filter? true}+ (meta/id :venues :id)]+ (#'sql.params.substitution/field->field-ref+ driver/*driver*+ (meta/field-metadata :venues :id)+ :number/=+ nil)))))(deftest ^:parallel honeysql->replacement-snippet-info-test(testing "make sure we handle quotes inside names correctly!"diff --git a/test/metabase/driver/sql/query_processor_test.clj b/test/metabase/driver/sql/query_processor_test.cljindex 7356f38d87e..d1b558747c6 100644--- a/test/metabase/driver/sql/query_processor_test.clj+++ b/test/metabase/driver/sql/query_processor_test.clj@@ -1400,7 +1400,7 @@(let [uuid #uuid "4f01dcfd-13f7-430c-8e6f-e505c0851027"](is (= uuid(sql.qp/->honeysql driver- [:value uuid {:base_type :type/UUID :effective_type :type/UUID}]))))))))+ (sql.qp/mbql-clause-with-opts driver :value {:base_type :type/UUID :effective_type :type/UUID} uuid)))))))))(deftest ^:parallel multiple-counts-test(testing "Count of count grouping works (#15074)"diff --git a/test/metabase/query_processor/timezones_test.clj b/test/metabase/query_processor/timezones_test.cljindex 60de1cc8c14..cf33ce1cd5f 100644--- a/test/metabase/query_processor/timezones_test.clj+++ b/test/metabase/query_processor/timezones_test.clj@@ -111,9 +111,10 @@(defn- field-identifier [table-key field-key](sql.qp/->honeysql driver/*driver*- [:field- (mt/id table-key field-key)- {:metabase.query-processor.util.add-alias-info/source-table (mt/id table-key)}]))+ (sql.qp/mbql-clause-with-opts driver/*driver*+ :field+ {:metabase.query-processor.util.add-alias-info/source-table (mt/id table-key)}+ (mt/id table-key field-key))))(defn- honeysql->sql [honeysql](first (sql.qp/format-honeysql driver/*driver* honeysql)))diff --git a/test/metabase/query_processor/util/relative_datetime_test.clj b/test/metabase/query_processor/util/relative_datetime_test.cljindex a934eda4d17..dfac49aea91 100644--- a/test/metabase/query_processor/util/relative_datetime_test.clj+++ b/test/metabase/query_processor/util/relative_datetime_test.clj@@ -25,9 +25,10 @@;; This was standard before PR #38604, now server side timestamps are used for that. This test confirms that;; server side generated timestamp (ie. new code path) results are equal to old code path results, that were not;; cacheable.- (let [honey {:select [[(mt/with-dynamic-fn-redefs [qp.relative-datetime/use-server-side-relative-datetime? (constantly false)]- (sql.qp/->honeysql driver/*driver* [:relative-datetime value unit]))]- [(sql.qp/->honeysql driver/*driver* [:relative-datetime value unit])]]}+ (let [rel-dt-clause (sql.qp/mbql-clause driver/*driver* :relative-datetime value unit)+ honey {:select [[(mt/with-dynamic-fn-redefs [qp.relative-datetime/use-server-side-relative-datetime? (constantly false)]+ (sql.qp/->honeysql driver/*driver* rel-dt-clause))]+ [(sql.qp/->honeysql driver/*driver* rel-dt-clause)]]}sql (sql/format honey)result (apply run-native-query sql)[db-generated ss-generated] (-> result mt/rows first)]diff --git a/test/metabase/test/data/postgres_mbql5.clj b/test/metabase/test/data/postgres_mbql5.cljnew file mode 100644index 00000000000..b04d8725fae--- /dev/null+++ b/test/metabase/test/data/postgres_mbql5.clj@@ -0,0 +1,2 @@+(ns metabase.test.data.postgres-mbql5+ "Placeholder; delete once mainline postgres does mbql5")